Split kafka-pr workflow outside of java-pr workflow (GoogleCloudPlatf…

…orm#1881)

.github/workflows/java-pr.yml

@@ -136,6 +136,7 @@ jobs:
       - name: Run Integration Smoke Tests
         run: | 
           ./cicd/run-it-smoke-tests \
+          --modules-to-build="DEFAULT" \
           --it-region="us-central1" \
           --it-project="cloud-teleport-testing" \
           --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \

.github/workflows/kafka-pr.yml

@@ -0,0 +1,207 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Checks that are intended to run on PRs containing Java code.
+
+name: Kafka PR
+
+on:
+  pull_request:
+    branches:
+    - 'main'
+    paths:
+      # Template wide common modules
+      - 'v2/common/**'
+      - 'v2/datastream-common/**'
+      - 'it/google-cloud-platform/**'
+      - 'it/conditions/**'
+      # Component common modules
+      - 'v2/kafka-common/**'
+      # kafka template paths
+      - '.github/workflows/kafka-pr.yml'
+      - 'v2/kafka-to-bigquery/**'
+      - 'v2/kafka-to-gcs/**'
+      - 'v2/kafka-to-kafka/**'
+  schedule:
+    - cron: "6 */12 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
+
+permissions: read-all
+
+jobs:
+  spotless_check:
+    name: Spotless
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Spotless
+        run: |
+          ./cicd/run-spotless \
+          --modules-to-build="KAFKA"
+  checkstyle_check:
+    name: Checkstyle
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Checkstyle
+        run: |
+          ./cicd/run-checkstyle \
+          --modules-to-build="KAFKA"
+  java_build:
+    name: Build
+    timeout-minutes: 60
+    runs-on: [self-hosted, it]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Build
+        run: |
+          ./cicd/run-build \
+          --modules-to-build="KAFKA"
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+  java_unit_tests:
+    name: Unit Tests
+    needs: [java_build]
+    timeout-minutes: 60
+    runs-on: [self-hosted, it]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Unit Tests
+        run: |
+          ./cicd/run-unit-tests \
+          --modules-to-build="KAFKA"
+      - name: Upload Unit Tests Report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: always() # always run even if the previous step fails
+        with:
+          name: surefire-test-results
+          path: '**/surefire-reports/TEST-*.xml'
+          retention-days: 1
+      - name: Upload coverage reports to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: GoogleCloudPlatform/DataflowTemplates
+          files: 'target/site/jacoco-aggregate/jacoco.xml'
+          # Temp fix for https://github.com/codecov/codecov-action/issues/1487
+          version: v0.6.0
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+  java_integration_smoke_tests_templates:
+    name: Dataflow Templates Integration Smoke Tests
+    needs: [spotless_check, checkstyle_check, java_build, java_unit_tests]
+    timeout-minutes: 60
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, it]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Integration Smoke Tests
+        run: |
+          ./cicd/run-it-smoke-tests \
+          --modules-to-build="KAFKA" \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1"
+      - name: Upload Smoke Tests Report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: always() # always run even if the previous step fails
+        with:
+          name: surefire-test-results
+          path: '**/surefire-reports/TEST-*.xml'
+          retention-days: 1
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+  java_integration_tests_templates:
+    name: Dataflow Templates Integration Tests
+    needs: [java_integration_smoke_tests_templates]
+    timeout-minutes: 240
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, it]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Integration Tests
+        run: |
+          ./cicd/run-it-tests \
+          --modules-to-build="KAFKA" \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1"
+      - name: Upload Integration Tests Report
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: always() # always run even if the previous step fails
+        with:
+          name: surefire-test-results
+          path: '**/surefire-reports/TEST-*.xml'
+          retention-days: 1
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()
+  java_load_tests_templates:
+    if: contains(github.event.pull_request.labels.*.name, 'run-load-tests')
+    name: Dataflow Templates Load Tests
+    needs: [spotless_check, checkstyle_check, java_build, java_integration_tests_templates]
+    timeout-minutes: 600
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [self-hosted, perf]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+      - name: Setup Environment
+        id: setup-env
+        uses: ./.github/actions/setup-env
+      - name: Run Load Tests
+        run: |
+          ./cicd/run-load-tests \
+          --it-region="us-central1" \
+          --it-project="cloud-teleport-testing" \
+          --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+          --it-private-connectivity="datastream-private-connect-us-central1"
+      - name: Cleanup Java Environment
+        uses: ./.github/actions/cleanup-java-env
+        if: always()

.github/workflows/spanner-pr.yml

@@ -38,11 +38,11 @@ on:
       - 'v2/sourcedb-to-spanner/**'
       - 'v2/spanner-to-sourcedb/**'
   schedule:
-  - cron: "0 */12 * * *"
+  - cron: "3 */12 * * *"
   workflow_dispatch:
 
 concurrency:
-  group: java-pr-${{ github.event.issue.number || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}
   cancel-in-progress: true
 
 env:

cicd/internal/flags/common-flags.go

@@ -24,6 +24,7 @@ import (
 const (
 	ALL     = "ALL"     // All modules
 	DEFAULT = "DEFAULT" // Modules other than those excluded
+	KAFKA   = "KAFKA"
 	SPANNER = "SPANNER"
 )
 
@@ -33,6 +34,13 @@ var (
 	moduleMap      = map[string][]string{
 		ALL:     {},
 		DEFAULT: {},
+		KAFKA: {"v2/kafka-common/",
+			"v2/kafka-to-bigquery/",
+			"v2/kafka-to-gcs/",
+			"v2/kafka-to-kafka/",
+			"v2/kafka-to-pubsub/",
+			"plugins/templates-maven-plugin",
+		},
 		SPANNER: {"v2/datastream-to-spanner/",
 			"v2/spanner-change-streams-to-sharded-file-sink/",
 			"v2/gcs-to-sourcedb/",
@@ -57,7 +65,7 @@ func ModulesToBuild() []string {
 		for k, v := range moduleMap {
 			if k != "ALL" && k != "DEFAULT" {
 				for _, n := range v {
-					if !strings.HasPrefix(n, "plugins/") {
+					if !(strings.HasPrefix(n, "plugins/") || strings.Contains(n, "common/")) {
 						s = append(s, "!"+n)
 					}
 				}

cicd/internal/flags/common-flags_test.go

@@ -18,8 +18,10 @@ package flags
 
 import (
 	"reflect"
-	"strings"
 	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 )
 
 func TestModulesToBuild(t *testing.T) {
@@ -39,6 +41,10 @@ func TestModulesToBuild(t *testing.T) {
 			input:    "ALL",
 			expected: []string{},
 		},
+		{
+			input:    "KAFKA",
+			expected: []string{"v2/kafka-common/", "v2/kafka-to-bigquery/", "v2/kafka-to-gcs/", "v2/kafka-to-kafka/", "v2/kafka-to-pubsub/", "plugins/templates-maven-plugin"},
+		},
 		{
 			input:    "SPANNER",
 			expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "v2/spanner-to-sourcedb/", "v2/spanner-custom-shard", "plugins/templates-maven-plugin"},
@@ -58,18 +64,23 @@ func TestDefaultExcludedSubModules(t *testing.T) {
 	// common modules won't excluded
 	modulesToBuild = "DEFAULT"
 	defaults := ModulesToBuild()
-	mods := []string{"SPANNER"}
+	// these are modules appended to moduleMap
+	excluded := map[string]int{"plugins/templates-maven-plugin": 0, "metadata/": 0, "v2/kafka-common/": 0}
 	var s []string
-	for _, m := range mods {
+	for m, _ := range moduleMap {
+		if m == "ALL" || m == "DEFAULT" {
+			continue
+		}
 		modulesToBuild = m
 		ms := ModulesToBuild()
 		for _, n := range ms {
-			if !strings.HasPrefix(n, "plugins/") {
+			if _, ok := excluded[n]; !ok {
 				s = append(s, "!"+n)
 			}
 		}
 	}
-	if !reflect.DeepEqual(defaults, s) {
+	less := func(a, b string) bool { return a < b }
+	if "" != cmp.Diff(defaults, s, cmpopts.SortSlices(less)) {
 		t.Errorf("Returned modules are not equal. Expected %v. Got %v.", s, defaults)
 	}
 }

v2/kafka-common/pom.xml

@@ -24,10 +24,6 @@
   <artifactId>kafka-common</artifactId>
   <modelVersion>4.0.0</modelVersion>
 
-  <properties>
-    <kafka-clients.version>3.7.0</kafka-clients.version>
-  </properties>
-
   <dependencies>
     <dependency>
       <groupId>com.google.cloud.teleport.v2</groupId>

v2/pom.xml

@@ -37,6 +37,7 @@
     <properties>
         <autovalue.service.version>1.0-rc6</autovalue.service.version>
         <jib.maven.version>3.4.0</jib.maven.version>
+        <kafka-clients.version>3.7.0</kafka-clients.version>
         <opencensus.version>0.31.0</opencensus.version>
         <protoc.version>3.21.12</protoc.version>
 

v2/streaming-data-generator/pom.xml

@@ -34,7 +34,11 @@
    </properties>
 
   <dependencies>
-
+    <dependency>
+      <groupId>com.google.cloud.teleport.metadata</groupId>
+      <artifactId>metadata</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.google.cloud.teleport.v2</groupId>
       <artifactId>common</artifactId>
@@ -45,12 +49,23 @@
       <artifactId>jdbc-common</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <!-- Kafka -->
     <dependency>
-      <groupId>com.google.cloud.teleport.v2</groupId>
-      <artifactId>kafka-common</artifactId>
-      <version>${project.version}</version>
+      <groupId>org.apache.beam</groupId>
+      <artifactId>beam-sdks-java-io-kafka</artifactId>
+      <version>${beam.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>${kafka-clients.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.xerial.snappy</groupId>
+          <artifactId>snappy-java</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
-
     <!-- Json -->
     <dependency>
       <groupId>com.github.vincentrussell</groupId>