diff --git a/.github/workflows/uw-tests.yml b/.github/workflows/uw-tests.yml new file mode 100644 index 0000000000..2a0f1b9c6f --- /dev/null +++ b/.github/workflows/uw-tests.yml @@ -0,0 +1,61 @@ +name: Release + +on: + workflow_dispatch: + inputs: + unifiedWorkerHarnessContainerImage: + description: 'Unified worker harness container image' + type: string + required: true + +permissions: + contents: write + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +jobs: + run_uw_tests: + name: Runner V2 Tests + runs-on: [self-hosted, release] + steps: + - name: Get releaser identity + run: | + git config --global user.name '${{github.actor}}' + git config --global user.email '${{github.actor}}@users.noreply.github.com' + # Defines releaseTag as latest successful release to avoid running tests failing at HEAD + - name: Declare runner image and release tag + id: variables + run: | + echo "unifiedWorkerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT + echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT + env: + HARNESS_IMAGE: ${{ inputs.unifiedWorkerHarnessContainerImage }} + - name: Checkout code + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 + with: + ref: ${{ steps.variables.outputs.releaseTag }} + token: ${{ secrets.RELEASE_TOKEN }} + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: ./cicd/run-build + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://batch-spanner.googleapis.com" \ + --it-release=true \ + --it-unified-worker-harness-container-image=$HARNESS_IMAGE \ + --it-retry-failures=2 + env: + HARNESS_IMAGE: ${{ steps.variables.outputs.unifiedWorkerHarnessContainerImage }} + - name: Upload Site Report + uses: ./.github/actions/publish-site-report + with: + output-zip-file: test-report + if: always() diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 45de079cfe..3a36246ef9 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -73,7 +73,8 @@ func main() { flags.CloudProxyHost(), flags.CloudProxyMySqlPort(), flags.CloudProxyPostgresPort(), - flags.CloudProxyPassword()) + flags.CloudProxyPassword(), + flags.UnifiedWorkerHarnessContainerImage()) if err != nil { log.Fatalf("%v\n", err) } diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 7d287c821b..176dbf83da 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -24,21 +24,22 @@ import ( // Avoid making these vars public. var ( - dRegion string - dProject string - dArtifactBucket string - dStageBucket string - dHostIp string - dPrivateConnectivity string - dSpannerHost string - dReleaseMode bool - dRetryFailures string - dCloudProxyHost string - dCloudProxyMySqlPort string - dCloudProxyPostgresPort string - dCloudProxyPassword string - dOracleHost string - dCloudOracleSysPassword string + dRegion string + dProject string + dArtifactBucket string + dStageBucket string + dHostIp string + dPrivateConnectivity string + dSpannerHost string + dReleaseMode bool + dRetryFailures string + dCloudProxyHost string + dCloudProxyMySqlPort string + dCloudProxyPostgresPort string + dCloudProxyPassword string + dOracleHost string + dCloudOracleSysPassword string + dUnifiedWorkerHarnessContainerImage string ) // Registers all it flags. Must be called before flag.Parse(). @@ -58,6 +59,7 @@ func RegisterItFlags() { flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy") flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB") flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB") + flag.StringVar(&dUnifiedWorkerHarnessContainerImage, "it-unified-worker-harness-container-image", "", "Runner harness image to run tests against") } func Region() string { @@ -142,3 +144,10 @@ func StaticOracleHost() string { func StaticOracleSysPassword() string { return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword } + +func UnifiedWorkerHarnessContainerImage() string { + if dUnifiedWorkerHarnessContainerImage != "" { + return "-Duw_staging_experiments=true -DunifiedWorker=true -DunifiedWorkerHarnessContainerImage=" + dUnifiedWorkerHarnessContainerImage + } + return "" +} diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java index d51d428c1a..d1b0425016 100644 --- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java +++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java @@ -493,12 +493,24 @@ protected LaunchInfo launchTemplate( if (System.getProperty("sdkContainerImage") != null) { options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage")); + } + if (System.getProperty("unifiedWorkerHarnessContainerImage") != null) { appendExperiment( - options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage")); - appendExperiment(options, "disable_worker_rolling_upgrade"); + options, + "runner_harness_container_image=" + + System.getProperty("unifiedWorkerHarnessContainerImage")); } } + if (System.getProperty("uw_staging_experiments") != null) { + appendExperiment(options, "disable_worker_rolling_upgrade"); + appendExperiment(options, "use_beam_bq_sink"); + appendExperiment(options, "beam_fn_api"); + appendExperiment(options, "use_unified_worker"); + appendExperiment(options, "use_portable_job_submission"); + appendExperiment(options, "worker_region=" + REGION); + } + if (System.getProperty("enableCleanupState") != null) { appendExperiment(options, "enable_cleanup_state"); }