From eb1d108bfb87b08cf08eed3311ee5995a099c3c0 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Fri, 20 Sep 2024 10:30:05 -0400 Subject: [PATCH] Split kafka-pr workflow outside of java-pr workflow (#1881) --- .github/workflows/java-pr.yml | 1 + .github/workflows/kafka-pr.yml | 207 +++++++++++++++++++++++ .github/workflows/spanner-pr.yml | 4 +- cicd/internal/flags/common-flags.go | 10 +- cicd/internal/flags/common-flags_test.go | 21 ++- v2/kafka-common/pom.xml | 4 - v2/pom.xml | 1 + v2/streaming-data-generator/pom.xml | 25 ++- 8 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/kafka-pr.yml diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index 6a0ebe94c7..a777e95712 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -136,6 +136,7 @@ jobs: - name: Run Integration Smoke Tests run: | ./cicd/run-it-smoke-tests \ + --modules-to-build="DEFAULT" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ diff --git a/.github/workflows/kafka-pr.yml b/.github/workflows/kafka-pr.yml new file mode 100644 index 0000000000..e293699bd8 --- /dev/null +++ b/.github/workflows/kafka-pr.yml @@ -0,0 +1,207 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that are intended to run on PRs containing Java code. + +name: Kafka PR + +on: + pull_request: + branches: + - 'main' + paths: + # Template wide common modules + - 'v2/common/**' + - 'v2/datastream-common/**' + - 'it/google-cloud-platform/**' + - 'it/conditions/**' + # Component common modules + - 'v2/kafka-common/**' + # kafka template paths + - '.github/workflows/kafka-pr.yml' + - 'v2/kafka-to-bigquery/**' + - 'v2/kafka-to-gcs/**' + - 'v2/kafka-to-kafka/**' + schedule: + - cron: "6 */12 * * *" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +permissions: read-all + +jobs: + spotless_check: + name: Spotless + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Spotless + run: | + ./cicd/run-spotless \ + --modules-to-build="KAFKA" + checkstyle_check: + name: Checkstyle + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Checkstyle + run: | + ./cicd/run-checkstyle \ + --modules-to-build="KAFKA" + java_build: + name: Build + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: | + ./cicd/run-build \ + --modules-to-build="KAFKA" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_unit_tests: + name: Unit Tests + needs: [java_build] + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Unit Tests + run: | + ./cicd/run-unit-tests \ + --modules-to-build="KAFKA" + - name: Upload Unit Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: GoogleCloudPlatform/DataflowTemplates + files: 'target/site/jacoco-aggregate/jacoco.xml' + # Temp fix for https://github.com/codecov/codecov-action/issues/1487 + version: v0.6.0 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_smoke_tests_templates: + name: Dataflow Templates Integration Smoke Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] + timeout-minutes: 60 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Smoke Tests + run: | + ./cicd/run-it-smoke-tests \ + --modules-to-build="KAFKA" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Smoke Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_tests_templates: + name: Dataflow Templates Integration Tests + needs: [java_integration_smoke_tests_templates] + timeout-minutes: 240 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --modules-to-build="KAFKA" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + if: always() + java_load_tests_templates: + if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') + name: Dataflow Templates Load Tests + needs: [spotless_check, checkstyle_check, java_build, java_integration_tests_templates] + timeout-minutes: 600 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, perf] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Load Tests + run: | + ./cicd/run-load-tests \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + if: always() diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index d9df005f2a..4a92edf4e7 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -38,11 +38,11 @@ on: - 'v2/sourcedb-to-spanner/**' - 'v2/spanner-to-sourcedb/**' schedule: - - cron: "0 */12 * * *" + - cron: "3 */12 * * *" workflow_dispatch: concurrency: - group: java-pr-${{ github.event.issue.number || github.run_id }} + group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }} cancel-in-progress: true env: diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index fab3276ba1..a7456396fd 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -24,6 +24,7 @@ import ( const ( ALL = "ALL" // All modules DEFAULT = "DEFAULT" // Modules other than those excluded + KAFKA = "KAFKA" SPANNER = "SPANNER" ) @@ -33,6 +34,13 @@ var ( moduleMap = map[string][]string{ ALL: {}, DEFAULT: {}, + KAFKA: {"v2/kafka-common/", + "v2/kafka-to-bigquery/", + "v2/kafka-to-gcs/", + "v2/kafka-to-kafka/", + "v2/kafka-to-pubsub/", + "plugins/templates-maven-plugin", + }, SPANNER: {"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", @@ -57,7 +65,7 @@ func ModulesToBuild() []string { for k, v := range moduleMap { if k != "ALL" && k != "DEFAULT" { for _, n := range v { - if !strings.HasPrefix(n, "plugins/") { + if !(strings.HasPrefix(n, "plugins/") || strings.Contains(n, "common/")) { s = append(s, "!"+n) } } diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 91df69626f..b60001d918 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -18,8 +18,10 @@ package flags import ( "reflect" - "strings" "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" ) func TestModulesToBuild(t *testing.T) { @@ -39,6 +41,10 @@ func TestModulesToBuild(t *testing.T) { input: "ALL", expected: []string{}, }, + { + input: "KAFKA", + expected: []string{"v2/kafka-common/", "v2/kafka-to-bigquery/", "v2/kafka-to-gcs/", "v2/kafka-to-kafka/", "v2/kafka-to-pubsub/", "plugins/templates-maven-plugin"}, + }, { input: "SPANNER", expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "v2/spanner-to-sourcedb/", "v2/spanner-custom-shard", "plugins/templates-maven-plugin"}, @@ -58,18 +64,23 @@ func TestDefaultExcludedSubModules(t *testing.T) { // common modules won't excluded modulesToBuild = "DEFAULT" defaults := ModulesToBuild() - mods := []string{"SPANNER"} + // these are modules appended to moduleMap + excluded := map[string]int{"plugins/templates-maven-plugin": 0, "metadata/": 0, "v2/kafka-common/": 0} var s []string - for _, m := range mods { + for m, _ := range moduleMap { + if m == "ALL" || m == "DEFAULT" { + continue + } modulesToBuild = m ms := ModulesToBuild() for _, n := range ms { - if !strings.HasPrefix(n, "plugins/") { + if _, ok := excluded[n]; !ok { s = append(s, "!"+n) } } } - if !reflect.DeepEqual(defaults, s) { + less := func(a, b string) bool { return a < b } + if "" != cmp.Diff(defaults, s, cmpopts.SortSlices(less)) { t.Errorf("Returned modules are not equal. Expected %v. Got %v.", s, defaults) } } diff --git a/v2/kafka-common/pom.xml b/v2/kafka-common/pom.xml index 8eca70cb6c..462cef4b15 100644 --- a/v2/kafka-common/pom.xml +++ b/v2/kafka-common/pom.xml @@ -24,10 +24,6 @@ kafka-common 4.0.0 - - 3.7.0 - - com.google.cloud.teleport.v2 diff --git a/v2/pom.xml b/v2/pom.xml index b8001738fe..ae9308ea27 100644 --- a/v2/pom.xml +++ b/v2/pom.xml @@ -37,6 +37,7 @@ 1.0-rc6 3.4.0 + 3.7.0 0.31.0 3.21.12 diff --git a/v2/streaming-data-generator/pom.xml b/v2/streaming-data-generator/pom.xml index 65873dee3e..8fe6f2d23c 100644 --- a/v2/streaming-data-generator/pom.xml +++ b/v2/streaming-data-generator/pom.xml @@ -34,7 +34,11 @@ - + + com.google.cloud.teleport.metadata + metadata + ${project.version} + com.google.cloud.teleport.v2 common @@ -45,12 +49,23 @@ jdbc-common ${project.version} + - com.google.cloud.teleport.v2 - kafka-common - ${project.version} + org.apache.beam + beam-sdks-java-io-kafka + ${beam.version} + + + org.apache.kafka + kafka-clients + ${kafka-clients.version} + + + org.xerial.snappy + snappy-java + + - com.github.vincentrussell