Add workflow for runner_v2 tests (GoogleCloudPlatform#1875)

* Add workflow for runner_v2 tests Signed-off-by: Jeffrey Kinard <[email protected]> * Rename runner to unified worker Signed-off-by: Jeffrey Kinard <[email protected]> --------- Signed-off-by: Jeffrey Kinard <[email protected]>
dedocibula · Sep 20, 2024 · 58be5c5 · 58be5c5
1 parent eb1d108
commit 58be5c5
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 18 deletions.
diff --git a/.github/workflows/uw-tests.yml b/.github/workflows/uw-tests.yml
@@ -0,0 +1,61 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      unifiedWorkerHarnessContainerImage:
+        description: 'Unified worker harness container image'
+        type: string
+        required: true
+
+permissions:
+  contents: write
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+jobs:
+  run_uw_tests:
+    name: Runner V2 Tests
+    runs-on: [self-hosted, release]
+    steps:
+      - name: Get releaser identity
+        run: |
+          git config --global user.name '${{github.actor}}'
+          git config --global user.email '${{github.actor}}@users.noreply.github.com'
+      # Defines releaseTag as latest successful release to avoid running tests failing at HEAD
+      - name: Declare runner image and release tag
+        id: variables
+        run: |
+          echo "unifiedWorkerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT
+          echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT
+        env:
+          HARNESS_IMAGE: ${{ inputs.unifiedWorkerHarnessContainerImage }}
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ steps.variables.outputs.releaseTag }}
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Build
+        run: ./cicd/run-build
+      - name: Run Integration Tests
+        run: |
+          ./cicd/run-it-tests \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1" \
+          --it-spanner-host="https://batch-spanner.googleapis.com" \
+          --it-release=true \
+          --it-unified-worker-harness-container-image=$HARNESS_IMAGE \
+          --it-retry-failures=2
+        env:
+          HARNESS_IMAGE: ${{ steps.variables.outputs.unifiedWorkerHarnessContainerImage }}
+      - name: Upload Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: test-report
+        if: always()
diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go
@@ -73,7 +73,8 @@ func main() {
 		flags.CloudProxyHost(),
 		flags.CloudProxyMySqlPort(),
 		flags.CloudProxyPostgresPort(),
-		flags.CloudProxyPassword())
+		flags.CloudProxyPassword(),
+		flags.UnifiedWorkerHarnessContainerImage())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go
@@ -24,21 +24,22 @@ import (
 
 // Avoid making these vars public.
 var (
-	dRegion                 string
-	dProject                string
-	dArtifactBucket         string
-	dStageBucket            string
-	dHostIp                 string
-	dPrivateConnectivity    string
-	dSpannerHost            string
-	dReleaseMode            bool
-	dRetryFailures          string
-	dCloudProxyHost         string
-	dCloudProxyMySqlPort    string
-	dCloudProxyPostgresPort string
-	dCloudProxyPassword     string
-	dOracleHost             string
-	dCloudOracleSysPassword string
+	dRegion                             string
+	dProject                            string
+	dArtifactBucket                     string
+	dStageBucket                        string
+	dHostIp                             string
+	dPrivateConnectivity                string
+	dSpannerHost                        string
+	dReleaseMode                        bool
+	dRetryFailures                      string
+	dCloudProxyHost                     string
+	dCloudProxyMySqlPort                string
+	dCloudProxyPostgresPort             string
+	dCloudProxyPassword                 string
+	dOracleHost                         string
+	dCloudOracleSysPassword             string
+	dUnifiedWorkerHarnessContainerImage string
 )
 
 // Registers all it flags. Must be called before flag.Parse().
@@ -58,6 +59,7 @@ func RegisterItFlags() {
 	flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy")
 	flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB")
 	flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB")
+	flag.StringVar(&dUnifiedWorkerHarnessContainerImage, "it-unified-worker-harness-container-image", "", "Runner harness image to run tests against")
 }
 
 func Region() string {
@@ -142,3 +144,10 @@ func StaticOracleHost() string {
 func StaticOracleSysPassword() string {
 	return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword
 }
+
+func UnifiedWorkerHarnessContainerImage() string {
+	if dUnifiedWorkerHarnessContainerImage != "" {
+		return "-Duw_staging_experiments=true -DunifiedWorker=true -DunifiedWorkerHarnessContainerImage=" + dUnifiedWorkerHarnessContainerImage
+	}
+	return ""
+}
diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java
@@ -493,12 +493,24 @@ protected LaunchInfo launchTemplate(
 
       if (System.getProperty("sdkContainerImage") != null) {
         options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage"));
+      }
+      if (System.getProperty("unifiedWorkerHarnessContainerImage") != null) {
         appendExperiment(
-            options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage"));
-        appendExperiment(options, "disable_worker_rolling_upgrade");
+            options,
+            "runner_harness_container_image="
+                + System.getProperty("unifiedWorkerHarnessContainerImage"));
       }
     }
 
+    if (System.getProperty("uw_staging_experiments") != null) {
+      appendExperiment(options, "disable_worker_rolling_upgrade");
+      appendExperiment(options, "use_beam_bq_sink");
+      appendExperiment(options, "beam_fn_api");
+      appendExperiment(options, "use_unified_worker");
+      appendExperiment(options, "use_portable_job_submission");
+      appendExperiment(options, "worker_region=" + REGION);
+    }
+
     if (System.getProperty("enableCleanupState") != null) {
       appendExperiment(options, "enable_cleanup_state");
     }