Add workflow for runner_v2 tests

Signed-off-by: Jeffrey Kinard <[email protected]>

.github/workflows/java-pr.yml

@@ -14,6 +14,7 @@
 
 # Checks that are intended to run on PRs containing Java code.
 
+
 name: Java PR
 
 on:
@@ -30,176 +31,55 @@ on:
       # This will make it easier to verify action changes don't break anything.
       - '.github/actions/setup-env/*'
       - '.github/workflows/java-pr.yml'
-      # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml)
-      - '!v2/datastream-to-spanner/**'
-      - '!v2/spanner-common/**'
-      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
-      - '!v2/gcs-to-sourcedb/**'
-      - '!v2/spanner-migrations-sdk/**'
-      - '!v2/spanner-custom-shard/**'
-      - '!v2/sourcedb-to-spanner/**'
   schedule:
     - cron: "0 */12 * * *"
   workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-
 env:
   MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
-
 permissions: read-all
 
 jobs:
-  spotless_check:
-    name: Spotless
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Spotless
-        run: ./cicd/run-spotless
-  checkstyle_check:
-    name: Checkstyle
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Checkstyle
-        run: ./cicd/run-checkstyle
-  java_build:
-    name: Build
-    timeout-minutes: 60
+  release:
+    name: Run Tests
     runs-on: [self-hosted, it]
     steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Get releaser identity
+        run: |
+          git config --global user.name '${{github.actor}}'
+          git config --global user.email '${{github.actor}}@users.noreply.github.com'
+      - name: Declare runner image and release tag
+        id: variables
+        run: |
+          echo "runnerHarnessContainerImage=gcr.io/dataflow-build/daily/unified-harness:20240916-02-rc00" >> $GITHUB_OUTPUT
+          echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ steps.variables.outputs.releaseTag }}
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Build
         run: ./cicd/run-build
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-  java_unit_tests:
-    name: Unit Tests
-    needs: [java_build]
-    timeout-minutes: 60
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Unit Tests
-        run: ./cicd/run-unit-tests
-      - name: Upload Unit Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Upload coverage reports to Codecov
-        uses: codecov/[email protected]
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          slug: GoogleCloudPlatform/DataflowTemplates
-          files: 'target/site/jacoco-aggregate/jacoco.xml'
-          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
-          version: v0.6.0
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_smoke_tests_templates:
-    name: Dataflow Templates Integration Smoke Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
-    timeout-minutes: 60
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
-      - name: Run Integration Smoke Tests
-        run: | 
-          ./cicd/run-it-smoke-tests \
-          --it-region="us-central1" \
-          --it-project="cloud-teleport-testing" \
-          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
-      - name: Upload Smoke Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
-        with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
-        if: always()
-  java_integration_tests_templates:
-    name: Dataflow Templates Integration Tests
-    needs: [java_integration_smoke_tests_templates]
-    timeout-minutes: 240
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, it]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - name: Setup Environment
-        id: setup-env
-        uses: ./.github/actions/setup-env
       - name: Run Integration Tests
-        run: | 
+        run: |
           ./cicd/run-it-tests \
-          --modules-to-build="DEFAULT" \
           --it-region="us-central1" \
           --it-project="cloud-teleport-testing" \
           --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-          --it-private-connectivity="datastream-private-connect-us-central1"
-      - name: Upload Integration Tests Report
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        if: always() # always run even if the previous step fails
+          --it-private-connectivity="datastream-private-connect-us-central1" \
+          --it-spanner-host="https://batch-spanner.googleapis.com" \
+          --it-release=true \
+          --it-runner-harness-container-image=$HARNESS_IMAGE \
+          --it-retry-failures=2
+        env:
+          HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }}
+      - name: Upload Site Report
+        uses: ./.github/actions/publish-site-report
         with:
-          name: surefire-test-results
-          path: '**/surefire-reports/TEST-*.xml'
-          retention-days: 1
-      - name: Cleanup Java Environment
-        uses: ./.github/actions/cleanup-java-env
+          output-zip-file: test-report
         if: always()
-  java_load_tests_templates:
-    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
-    name: Dataflow Templates Load Tests
-    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates]
-    timeout-minutes: 600
-    # Run on any runner that matches all the specified runs-on values.
-    runs-on: [self-hosted, perf]
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-    - name: Setup Environment
-      id: setup-env
-      uses: ./.github/actions/setup-env
-    - name: Run Load Tests
-      run: |
-        ./cicd/run-load-tests \
-        --it-region="us-central1" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-private-connect-us-central1"
-    - name: Cleanup Java Environment
-      uses: ./.github/actions/cleanup-java-env
-      if: always()

.github/workflows/uw-tests.yml

@@ -0,0 +1,60 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      runnerHarnessContainerImage:
+        description: 'Runner harness container image'
+        type: string
+        required: true
+
+permissions:
+  contents: write
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+jobs:
+  release:
+    name: Runner V2 Tests
+    runs-on: [self-hosted, release]
+    steps:
+      - name: Get releaser identity
+        run: |
+          git config --global user.name '${{github.actor}}'
+          git config --global user.email '${{github.actor}}@users.noreply.github.com'
+      - name: Declare runner image and release tag
+        id: variables
+        run: |
+          echo "runnerHarnessContainerImage=${HARNESS_IMAGE}" >> $GITHUB_OUTPUT
+          echo "releaseTag=$(curl -s https://api.github.com/repos/GoogleCLoudPlatform/DataflowTemplates/releases/latest | jq '.tag_name' | sed 's/\"//g')" >> $GITHUB_OUTPUT
+        env:
+          HARNESS_IMAGE: ${{ inputs.runnerHarnessContainerImage }}
+      - name: Checkout code
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
+        with:
+          ref: ${{ steps.variables.outputs.releaseTag }}
+          token: ${{ secrets.RELEASE_TOKEN }}
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Build
+        run: ./cicd/run-build
+      - name: Run Integration Tests
+        run: |
+          ./cicd/run-it-tests \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1" \
+          --it-spanner-host="https://batch-spanner.googleapis.com" \
+          --it-release=true \
+          --it-runner-harness-container-image=$HARNESS_IMAGE \
+          --it-retry-failures=2
+        env:
+          HARNESS_IMAGE: ${{ steps.variables.outputs.runnerHarnessContainerImage }}
+      - name: Upload Site Report
+        uses: ./.github/actions/publish-site-report
+        with:
+          output-zip-file: test-report
+        if: always()

cicd/cmd/run-it-tests/main.go

@@ -73,7 +73,8 @@ func main() {
 		flags.CloudProxyHost(),
 		flags.CloudProxyMySqlPort(),
 		flags.CloudProxyPostgresPort(),
-		flags.CloudProxyPassword())
+		flags.CloudProxyPassword(),
+		flags.RunnerHarnessContainerImage())
 	if err != nil {
 		log.Fatalf("%v\n", err)
 	}

cicd/internal/flags/it-flags.go

@@ -24,21 +24,22 @@ import (
 
 // Avoid making these vars public.
 var (
-	dRegion                 string
-	dProject                string
-	dArtifactBucket         string
-	dStageBucket            string
-	dHostIp                 string
-	dPrivateConnectivity    string
-	dSpannerHost            string
-	dReleaseMode            bool
-	dRetryFailures          string
-	dCloudProxyHost         string
-	dCloudProxyMySqlPort    string
-	dCloudProxyPostgresPort string
-	dCloudProxyPassword     string
-	dOracleHost             string
-	dCloudOracleSysPassword string
+	dRegion                      string
+	dProject                     string
+	dArtifactBucket              string
+	dStageBucket                 string
+	dHostIp                      string
+	dPrivateConnectivity         string
+	dSpannerHost                 string
+	dReleaseMode                 bool
+	dRetryFailures               string
+	dCloudProxyHost              string
+	dCloudProxyMySqlPort         string
+	dCloudProxyPostgresPort      string
+	dCloudProxyPassword          string
+	dOracleHost                  string
+	dCloudOracleSysPassword      string
+	dRunnerHarnessContainerImage string
 )
 
 // Registers all it flags. Must be called before flag.Parse().
@@ -58,6 +59,7 @@ func RegisterItFlags() {
 	flag.StringVar(&dCloudProxyPassword, "it-cloud-proxy-password", "t>5xl%J(&qTK6?FaZ", "Password of static Cloud Auth Proxy")
 	flag.StringVar(&dOracleHost, "it-oracle-host", "10.128.0.90", "Hostname or IP address of static Oracle DB")
 	flag.StringVar(&dCloudOracleSysPassword, "it-oracle-sys-password", "oracle", "sys password of static Oracle DB")
+	flag.StringVar(&dRunnerHarnessContainerImage, "it-runner-harness-container-image", "", "Runner harness image to run tests against")
 }
 
 func Region() string {
@@ -142,3 +144,10 @@ func StaticOracleHost() string {
 func StaticOracleSysPassword() string {
 	return "-DcloudOracleSysPassword=" + dCloudOracleSysPassword
 }
+
+func RunnerHarnessContainerImage() string {
+	if dRunnerHarnessContainerImage != "" {
+		return "-Duw_staging_experiments=true -DunifiedWorker=true -DrunnerHarnessContainerImage=" + dRunnerHarnessContainerImage
+	}
+	return ""
+}

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java

@@ -493,12 +493,23 @@ protected LaunchInfo launchTemplate(
 
       if (System.getProperty("sdkContainerImage") != null) {
         options.addParameter("sdkContainerImage", System.getProperty("sdkContainerImage"));
+      }
+      if (System.getProperty("runnerHarnessContainerImage") != null) {
         appendExperiment(
-            options, "worker_harness_container_image=" + System.getProperty("sdkContainerImage"));
-        appendExperiment(options, "disable_worker_rolling_upgrade");
+            options,
+            "runner_harness_container_image=" + System.getProperty("runnerHarnessContainerImage"));
       }
     }
 
+    if (System.getProperty("uw_staging_experiments") != null) {
+      appendExperiment(options, "disable_worker_rolling_upgrade");
+      appendExperiment(options, "use_beam_bq_sink");
+      appendExperiment(options, "beam_fn_api");
+      appendExperiment(options, "use_unified_worker");
+      appendExperiment(options, "use_portable_job_submission");
+      appendExperiment(options, "worker_region=" + REGION);
+    }
+
     if (System.getProperty("enableCleanupState") != null) {
       appendExperiment(options, "enable_cleanup_state");
     }