From 8625934c0a799ba0e26f9810440bd186e12be5db Mon Sep 17 00:00:00 2001 From: Jeffrey Kinard Date: Tue, 17 Sep 2024 18:20:39 -0400 Subject: [PATCH 1/2] Add workflow for runner_v2 tests Signed-off-by: Jeffrey Kinard --- .github/workflows/uw-tests.yml | 61 +++++++++++++++++++ cicd/cmd/run-it-tests/main.go | 3 +- cicd/internal/flags/it-flags.go | 39 +++++++----- .../apache/beam/it/gcp/TemplateTestBase.java | 15 ++++- 4 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/uw-tests.yml diff --git a/.github/workflows/uw-tests.yml b/.github/workflows/uw-tests.yml new file mode 100644 index 0000000000..83f82d6ad4 --- /dev/null +++ b/.github/workflows/uw-tests.yml @@ -0,0 +1,61 @@ +name: Release + +on: + workflow_dispatch: + inputs: + runnerHarnessContainerImage: + description: 'Runner harness container image' + type: string + required: true + +permissions: + contents: write + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +jobs: + run_uw_tests: + name: Runner V2 Tests + runs-on: [self-hosted, release] + steps: + - name: Get releaser identity + run: | + git config --global user.name '${{github.actor}}' + git config --global user.email '${{github.actor}}@users.noreply.github.com' + # Defines releaseTag as latest successful release to avoid running tests failing at HEAD + - name: Declare runner image and release tag + id: variables + run: | + echo "runnerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT + echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT + env: + HARNESS_IMAGE: ${{ inputs.runnerHarnessContainerImage }} + - name: Checkout code + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 + with: + ref: ${{ steps.variables.outputs.releaseTag }} + token: ${{ secrets.RELEASE_TOKEN }} + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: ./cicd/run-build + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://batch-spanner.googleapis.com" \ + --it-release=true \ + --it-runner-harness-container-image=$HARNESS_IMAGE \ + --it-retry-failures=2 + env: + HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }} + - name: Upload Site Report + uses: ./.github/actions/publish-site-report + with: + output-zip-file: test-report + if: always() diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 45de079cfe..1ba45d0674 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -73,7 +73,8 @@ func main() { flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), - flags.CloudProxyPassword()) + flags.CloudProxyPassword(), + flags.RunnerHarnessContainerImage()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7d287c821b..7b499195db 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -24,21 +24,22 @@ import ( // Avoid making these vars public. var ( - dRegion string - dProject string - dArtifactBucket string - dStageBucket string - dHostIp string - dPrivateConnectivity string - dSpannerHost string - dReleaseMode bool - dRetryFailures string - dCloudProxyHost string - dCloudProxyMySqlPort string - dCloudProxyPostgresPort string - dCloudProxyPassword string - dOracleHost string - dCloudOracleSysPassword string + dRegion string + dProject string + dArtifactBucket string + dStageBucket string + dHostIp string + dPrivateConnectivity string + dSpannerHost string + dReleaseMode bool + dRetryFailures string + dCloudProxyHost string + dCloudProxyMySqlPort string + dCloudProxyPostgresPort string + dCloudProxyPassword string + dOracleHost string + dCloudOracleSysPassword string + dRunnerHarnessContainerImage string ) // Registers all it flags. Must be called before flag.Parse(). @@ -58,6 +59,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") + flag.StringVar(&dRunnerHarnessContainerImage, "it-runner-harness-container-image", "", "Runner harness image to run tests against") } func Region() string { @@ -142,3 +144,10 @@ func StaticOracleHost() string { func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } + +func RunnerHarnessContainerImage() string { + if dRunnerHarnessContainerImage != "" { + return "-Duw_staging_experiments=true -DunifiedWorker=true -DrunnerHarnessContainerImage=" + dRunnerHarnessContainerImage + } + return "" +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java index d51d428c1a..81ea8b2480 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java @@ -493,12 +493,23 @@ protected LaunchInfo launchTemplate( if (System.getProperty("sdkContainerImage") != null) { options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage")); + } + if (System.getProperty("runnerHarnessContainerImage") != null) { appendExperiment( - options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage")); - appendExperiment(options, "disable_worker_rolling_upgrade"); + options, + "runner_harness_container_image=" + System.getProperty("runnerHarnessContainerImage")); } } + if (System.getProperty("uw_staging_experiments") != null) { + appendExperiment(options, "disable_worker_rolling_upgrade"); + appendExperiment(options, "use_beam_bq_sink"); + appendExperiment(options, "beam_fn_api"); + appendExperiment(options, "use_unified_worker"); + appendExperiment(options, "use_portable_job_submission"); + appendExperiment(options, "worker_region=" + REGION); + } + if (System.getProperty("enableCleanupState") != null) { appendExperiment(options, "enable_cleanup_state"); } From 6b62b11dc40b1a4cd20d415a6fd9218950054bd9 Mon Sep 17 00:00:00 2001 From: Jeffrey Kinard Date: Fri, 20 Sep 2024 11:19:40 -0400 Subject: [PATCH 2/2] Rename runner to unified worker Signed-off-by: Jeffrey Kinard --- .github/workflows/uw-tests.yml | 12 +++--- cicd/cmd/run-it-tests/main.go | 2 +- cicd/internal/flags/it-flags.go | 40 +++++++++---------- .../apache/beam/it/gcp/TemplateTestBase.java | 5 ++- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/uw-tests.yml b/.github/workflows/uw-tests.yml index 83f82d6ad4..2a0f1b9c6f 100644 --- a/.github/workflows/uw-tests.yml +++ b/.github/workflows/uw-tests.yml @@ -3,8 +3,8 @@ name: Release on: workflow_dispatch: inputs: - runnerHarnessContainerImage: - description: 'Runner harness container image' + unifiedWorkerHarnessContainerImage: + description: 'Unified worker harness container image' type: string required: true @@ -27,10 +27,10 @@ jobs: - name: Declare runner image and release tag id: variables run: | - echo "runnerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT + echo "unifiedWorkerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT env: - HARNESS_IMAGE: ${{ inputs.runnerHarnessContainerImage }} + HARNESS_IMAGE: ${{ inputs.unifiedWorkerHarnessContainerImage }} - name: Checkout code uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 with: @@ -50,10 +50,10 @@ jobs: --it-private-connectivity="datastream-private-connect-us-central1" \ --it-spanner-host="https://batch-spanner.googleapis.com" \ --it-release=true \ - --it-runner-harness-container-image=$HARNESS_IMAGE \ + --it-unified-worker-harness-container-image=$HARNESS_IMAGE \ --it-retry-failures=2 env: - HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }} + HARNESS_IMAGE: ${{ steps.variables.outputs.unifiedWorkerHarnessContainerImage }} - name: Upload Site Report uses: ./.github/actions/publish-site-report with: diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 1ba45d0674..3a36246ef9 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -74,7 +74,7 @@ func main() { flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), flags.CloudProxyPassword(), - flags.RunnerHarnessContainerImage()) + flags.UnifiedWorkerHarnessContainerImage()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7b499195db..176dbf83da 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -24,22 +24,22 @@ import ( // Avoid making these vars public. var ( - dRegion string - dProject string - dArtifactBucket string - dStageBucket string - dHostIp string - dPrivateConnectivity string - dSpannerHost string - dReleaseMode bool - dRetryFailures string - dCloudProxyHost string - dCloudProxyMySqlPort string - dCloudProxyPostgresPort string - dCloudProxyPassword string - dOracleHost string - dCloudOracleSysPassword string - dRunnerHarnessContainerImage string + dRegion string + dProject string + dArtifactBucket string + dStageBucket string + dHostIp string + dPrivateConnectivity string + dSpannerHost string + dReleaseMode bool + dRetryFailures string + dCloudProxyHost string + dCloudProxyMySqlPort string + dCloudProxyPostgresPort string + dCloudProxyPassword string + dOracleHost string + dCloudOracleSysPassword string + dUnifiedWorkerHarnessContainerImage string ) // Registers all it flags. Must be called before flag.Parse(). @@ -59,7 +59,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") - flag.StringVar(&dRunnerHarnessContainerImage, "it-runner-harness-container-image", "", "Runner harness image to run tests against") + flag.StringVar(&dUnifiedWorkerHarnessContainerImage, "it-unified-worker-harness-container-image", "", "Runner harness image to run tests against") } func Region() string { @@ -145,9 +145,9 @@ func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } -func RunnerHarnessContainerImage() string { - if dRunnerHarnessContainerImage != "" { - return "-Duw_staging_experiments=true -DunifiedWorker=true -DrunnerHarnessContainerImage=" + dRunnerHarnessContainerImage +func UnifiedWorkerHarnessContainerImage() string { + if dUnifiedWorkerHarnessContainerImage != "" { + return "-Duw_staging_experiments=true -DunifiedWorker=true -DunifiedWorkerHarnessContainerImage=" + dUnifiedWorkerHarnessContainerImage } return "" } diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java index 81ea8b2480..d1b0425016 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java @@ -494,10 +494,11 @@ protected LaunchInfo launchTemplate( if (System.getProperty("sdkContainerImage") != null) { options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage")); } - if (System.getProperty("runnerHarnessContainerImage") != null) { + if (System.getProperty("unifiedWorkerHarnessContainerImage") != null) { appendExperiment( options, - "runner_harness_container_image=" + System.getProperty("runnerHarnessContainerImage")); + "runner_harness_container_image=" + + System.getProperty("unifiedWorkerHarnessContainerImage")); } }