From c2cb186bf20e93ea66f58f9538a431a3c05a8727 Mon Sep 17 00:00:00 2001 From: Jeffrey Kinard Date: Tue, 17 Sep 2024 18:20:39 -0400 Subject: [PATCH] Add workflow for runner_v2 tests Signed-off-by: Jeffrey Kinard --- .github/workflows/java-pr.yml | 174 +++--------------- .github/workflows/uw-tests.yml | 60 ++++++ cicd/cmd/run-it-tests/main.go | 3 +- cicd/internal/flags/it-flags.go | 39 ++-- .../apache/beam/it/gcp/TemplateTestBase.java | 15 +- 5 files changed, 126 insertions(+), 165 deletions(-) create mode 100644 .github/workflows/uw-tests.yml diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index 6a0ebe94c73..b78176b0c49 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -14,6 +14,7 @@ # Checks that are intended to run on PRs containing Java code. + name: Java PR on: @@ -30,14 +31,6 @@ on: # This will make it easier to verify action changes don't break anything. - '.github/actions/setup-env/*' - '.github/workflows/java-pr.yml' - # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml) - - '!v2/datastream-to-spanner/**' - - '!v2/spanner-common/**' - - '!v2/spanner-change-streams-to-sharded-file-sink/**' - - '!v2/gcs-to-sourcedb/**' - - '!v2/spanner-migrations-sdk/**' - - '!v2/spanner-custom-shard/**' - - '!v2/sourcedb-to-spanner/**' schedule: - cron: "0 */12 * * *" workflow_dispatch: @@ -45,161 +38,48 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - env: MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error - permissions: read-all jobs: - spotless_check: - name: Spotless - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Spotless - run: ./cicd/run-spotless - checkstyle_check: - name: Checkstyle - timeout-minutes: 10 - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Checkstyle - run: ./cicd/run-checkstyle - java_build: - name: Build - timeout-minutes: 60 + release: + name: Run Tests runs-on: [self-hosted, it] steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Get releaser identity + run: | + git config --global user.name '${{github.actor}}' + git config --global user.email '${{github.actor}}@users.noreply.github.com' + - name: Declare runner image and release tag + id: variables + run: | + echo "runnerHarnessContainerImage=gcr.io/dataflow-build/daily/unified-harness:20240916-02-rc00" >> $GITHUB_OUTPUT + echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT + - name: Checkout code + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 + with: + ref: ${{ steps.variables.outputs.releaseTag }} - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Build run: ./cicd/run-build - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - java_unit_tests: - name: Unit Tests - needs: [java_build] - timeout-minutes: 60 - runs-on: [self-hosted, it] - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Unit Tests - run: ./cicd/run-unit-tests - - name: Upload Unit Tests Report - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - if: always() # always run even if the previous step fails - with: - name: surefire-test-results - path: '**/surefire-reports/TEST-*.xml' - retention-days: 1 - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.0.1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: GoogleCloudPlatform/DataflowTemplates - files: 'target/site/jacoco-aggregate/jacoco.xml' - # Temp fix for https://github.com/codecov/codecov-action/issues/1487 - version: v0.6.0 - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - java_integration_smoke_tests_templates: - name: Dataflow Templates Integration Smoke Tests - needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] - timeout-minutes: 60 - # Run on any runner that matches all the specified runs-on values. - runs-on: [self-hosted, it] - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Integration Smoke Tests - run: | - ./cicd/run-it-smoke-tests \ - --it-region="us-central1" \ - --it-project="cloud-teleport-testing" \ - --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-private-connect-us-central1" - - name: Upload Smoke Tests Report - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - if: always() # always run even if the previous step fails - with: - name: surefire-test-results - path: '**/surefire-reports/TEST-*.xml' - retention-days: 1 - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() - java_integration_tests_templates: - name: Dataflow Templates Integration Tests - needs: [java_integration_smoke_tests_templates] - timeout-minutes: 240 - # Run on any runner that matches all the specified runs-on values. - runs-on: [self-hosted, it] - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - name: Run Integration Tests - run: | + run: | ./cicd/run-it-tests \ - --modules-to-build="DEFAULT" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-private-connect-us-central1" - - name: Upload Integration Tests Report - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - if: always() # always run even if the previous step fails + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://batch-spanner.googleapis.com" \ + --it-release=true \ + --it-runner-harness-container-image=$HARNESS_IMAGE \ + --it-retry-failures=2 + env: + HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }} + - name: Upload Site Report + uses: ./.github/actions/publish-site-report with: - name: surefire-test-results - path: '**/surefire-reports/TEST-*.xml' - retention-days: 1 - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env + output-zip-file: test-report if: always() - java_load_tests_templates: - if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') - name: Dataflow Templates Load Tests - needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates] - timeout-minutes: 600 - # Run on any runner that matches all the specified runs-on values. - runs-on: [self-hosted, perf] - steps: - - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - name: Setup Environment - id: setup-env - uses: ./.github/actions/setup-env - - name: Run Load Tests - run: | - ./cicd/run-load-tests \ - --it-region="us-central1" \ - --it-project="cloud-teleport-testing" \ - --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ - --it-private-connectivity="datastream-private-connect-us-central1" - - name: Cleanup Java Environment - uses: ./.github/actions/cleanup-java-env - if: always() diff --git a/.github/workflows/uw-tests.yml b/.github/workflows/uw-tests.yml new file mode 100644 index 00000000000..f244bf21c0d --- /dev/null +++ b/.github/workflows/uw-tests.yml @@ -0,0 +1,60 @@ +name: Release + +on: + workflow_dispatch: + inputs: + runnerHarnessContainerImage: + description: 'Runner harness container image' + type: string + required: true + +permissions: + contents: write + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +jobs: + release: + name: Runner V2 Tests + runs-on: [self-hosted, release] + steps: + - name: Get releaser identity + run: | + git config --global user.name '${{github.actor}}' + git config --global user.email '${{github.actor}}@users.noreply.github.com' + - name: Declare runner image and release tag + id: variables + run: | + echo "runnerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT + echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT + env: + HARNESS_IMAGE: ${{ inputs.runnerHarnessContainerImage }} + - name: Checkout code + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 + with: + ref: ${{ steps.variables.outputs.releaseTag }} + token: ${{ secrets.RELEASE_TOKEN }} + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: ./cicd/run-build + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://batch-spanner.googleapis.com" \ + --it-release=true \ + --it-runner-harness-container-image=$HARNESS_IMAGE \ + --it-retry-failures=2 + env: + HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }} + - name: Upload Site Report + uses: ./.github/actions/publish-site-report + with: + output-zip-file: test-report + if: always() diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 45de079cfe9..1ba45d0674b 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -73,7 +73,8 @@ func main() { flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), - flags.CloudProxyPassword()) + flags.CloudProxyPassword(), + flags.RunnerHarnessContainerImage()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7d287c821bf..7b499195dbd 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -24,21 +24,22 @@ import ( // Avoid making these vars public. var ( - dRegion string - dProject string - dArtifactBucket string - dStageBucket string - dHostIp string - dPrivateConnectivity string - dSpannerHost string - dReleaseMode bool - dRetryFailures string - dCloudProxyHost string - dCloudProxyMySqlPort string - dCloudProxyPostgresPort string - dCloudProxyPassword string - dOracleHost string - dCloudOracleSysPassword string + dRegion string + dProject string + dArtifactBucket string + dStageBucket string + dHostIp string + dPrivateConnectivity string + dSpannerHost string + dReleaseMode bool + dRetryFailures string + dCloudProxyHost string + dCloudProxyMySqlPort string + dCloudProxyPostgresPort string + dCloudProxyPassword string + dOracleHost string + dCloudOracleSysPassword string + dRunnerHarnessContainerImage string ) // Registers all it flags. Must be called before flag.Parse(). @@ -58,6 +59,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") + flag.StringVar(&dRunnerHarnessContainerImage, "it-runner-harness-container-image", "", "Runner harness image to run tests against") } func Region() string { @@ -142,3 +144,10 @@ func StaticOracleHost() string { func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } + +func RunnerHarnessContainerImage() string { + if dRunnerHarnessContainerImage != "" { + return "-Duw_staging_experiments=true -DunifiedWorker=true -DrunnerHarnessContainerImage=" + dRunnerHarnessContainerImage + } + return "" +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java index d51d428c1ac..81ea8b2480a 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java @@ -493,12 +493,23 @@ protected LaunchInfo launchTemplate( if (System.getProperty("sdkContainerImage") != null) { options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage")); + } + if (System.getProperty("runnerHarnessContainerImage") != null) { appendExperiment( - options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage")); - appendExperiment(options, "disable_worker_rolling_upgrade"); + options, + "runner_harness_container_image=" + System.getProperty("runnerHarnessContainerImage")); } } + if (System.getProperty("uw_staging_experiments") != null) { + appendExperiment(options, "disable_worker_rolling_upgrade"); + appendExperiment(options, "use_beam_bq_sink"); + appendExperiment(options, "beam_fn_api"); + appendExperiment(options, "use_unified_worker"); + appendExperiment(options, "use_portable_job_submission"); + appendExperiment(options, "worker_region=" + REGION); + } + if (System.getProperty("enableCleanupState") != null) { appendExperiment(options, "enable_cleanup_state"); }