Add workflow for runner_v2 tests

Signed-off-by: Jeffrey Kinard <[email protected]>

.github/workflows/uw-tests.yml

@@ -0,0 +1,61 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      runnerHarnessContainerImage:
+        description: 'Runner harness container image'
+        type: string
+        required: true
+
+permissions:
+  contents: write
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+jobs:
+  run_uw_tests:
+    name: Runner V2 Tests
+    runs-on: [self-hosted, release]
+    steps:
+      - name: Get releaser identity
+        run: |
+          git config --global user.name '${{github.actor}}'
+          git config --global user.email '${{github.actor}}@users.noreply.github.com'
+      # Defines releaseTag as latest successful release to avoid running tests failing at HEAD
+      - name: Declare runner image and release tag
+        id: variables
+        run: |
+          echo "runnerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT
+          echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT
+        env:
+          HARNESS_IMAGE: ${{ inputs.runnerHarnessContainerImage }}
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ steps.variables.outputs.releaseTag }}
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Build
+        run: ./cicd/run-build
+      - name: Run Integration Tests
+        run: |
+          ./cicd/run-it-tests \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1" \
+          --it-spanner-host="https://batch-spanner.googleapis.com" \
+          --it-release=true \
+          --it-runner-harness-container-image=$HARNESS_IMAGE \
+          --it-retry-failures=2
+        env:
+          HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }}
+      - name: Upload Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: test-report
+        if: always()

cicd/cmd/run-it-tests/main.go

@@ -73,7 +73,8 @@ func main() {
 		flags.CloudProxyHost(),
 		flags.CloudProxyMySqlPort(),
 		flags.CloudProxyPostgresPort(),
-		flags.CloudProxyPassword())
+		flags.CloudProxyPassword(),
+		flags.RunnerHarnessContainerImage())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/internal/flags/it-flags.go

@@ -24,21 +24,22 @@ import (
 
 // Avoid making these vars public.
 var (
-	dRegion                 string
-	dProject                string
-	dArtifactBucket         string
-	dStageBucket            string
-	dHostIp                 string
-	dPrivateConnectivity    string
-	dSpannerHost            string
-	dReleaseMode            bool
-	dRetryFailures          string
-	dCloudProxyHost         string
-	dCloudProxyMySqlPort    string
-	dCloudProxyPostgresPort string
-	dCloudProxyPassword     string
-	dOracleHost             string
-	dCloudOracleSysPassword string
+	dRegion                      string
+	dProject                     string
+	dArtifactBucket              string
+	dStageBucket                 string
+	dHostIp                      string
+	dPrivateConnectivity         string
+	dSpannerHost                 string
+	dReleaseMode                 bool
+	dRetryFailures               string
+	dCloudProxyHost              string
+	dCloudProxyMySqlPort         string
+	dCloudProxyPostgresPort      string
+	dCloudProxyPassword          string
+	dOracleHost                  string
+	dCloudOracleSysPassword      string
+	dRunnerHarnessContainerImage string
 )
 
 // Registers all it flags. Must be called before flag.Parse().
@@ -58,6 +59,7 @@ func RegisterItFlags() {
 	flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy")
 	flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB")
 	flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB")
+	flag.StringVar(&dRunnerHarnessContainerImage, "it-runner-harness-container-image", "", "Runner harness image to run tests against")
 }
 
 func Region() string {
@@ -142,3 +144,10 @@ func StaticOracleHost() string {
 func StaticOracleSysPassword() string {
 	return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword
 }
+
+func RunnerHarnessContainerImage() string {
+	if dRunnerHarnessContainerImage != "" {
+		return "-Duw_staging_experiments=true -DunifiedWorker=true -DrunnerHarnessContainerImage=" + dRunnerHarnessContainerImage
+	}
+	return ""
+}

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java

@@ -493,12 +493,23 @@ protected LaunchInfo launchTemplate(
 
       if (System.getProperty("sdkContainerImage") != null) {
         options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage"));
+      }
+      if (System.getProperty("runnerHarnessContainerImage") != null) {
         appendExperiment(
-            options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage"));
-        appendExperiment(options, "disable_worker_rolling_upgrade");
+            options,
+            "runner_harness_container_image=" + System.getProperty("runnerHarnessContainerImage"));
       }
     }
 
+    if (System.getProperty("uw_staging_experiments") != null) {
+      appendExperiment(options, "disable_worker_rolling_upgrade");
+      appendExperiment(options, "use_beam_bq_sink");
+      appendExperiment(options, "beam_fn_api");
+      appendExperiment(options, "use_unified_worker");
+      appendExperiment(options, "use_portable_job_submission");
+      appendExperiment(options, "worker_region=" + REGION);
+    }
+
     if (System.getProperty("enableCleanupState") != null) {
       appendExperiment(options, "enable_cleanup_state");
     }