diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 69ba7f1575..8ef59df42a 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -46,23 +46,15 @@ inputs: description: 'The version of Python to install' required: false default: '3.11' -outputs: - changed-files: - description: 'Comma-separated list of files that were changed' - value: ${{ steps.changed-files.outputs.all_changed_and_modified_files }} runs: using: 'composite' steps: - name: Setup Go - uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: ${{ inputs.go-version }} - - name: Get Changed Files - id: changed-files - uses: tj-actions/changed-files@61ee456a9d0f512e7ecfdf28863634c97dae2d16 - with: - separator: ',' + cache: false # It shouldn't take too long to build all of this, and it will at least # make running the target program easier - name: Build CI/CD diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml index d13a82281a..c3b50c7158 100644 --- a/.github/actions/setup-java-env/action.yml +++ b/.github/actions/setup-java-env/action.yml @@ -40,7 +40,7 @@ runs: using: 'composite' steps: - name: Setup Java - uses: actions/setup-java@a12e082d834968c1847f782019214fadd20719f6 + uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7 # v4.2.0 with: distribution: 'temurin' java-version: ${{ inputs.java-version }} @@ -57,7 +57,7 @@ runs: echo "YESTERDAY=$KEY" >> $GITHUB_ENV fi - name: Setup Cache - uses: actions/cache@72d1e4fdff0ff7b1b6e86b415f2d4f5941e5c006 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1 id: setup-cache with: path: | diff --git a/.github/scripts/configure-runners.sh b/.github/scripts/configure-runners.sh index eeee825d19..a51b9aaf7e 100755 --- a/.github/scripts/configure-runners.sh +++ b/.github/scripts/configure-runners.sh @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# For running the script, see go/templates-gitactions-script +# For running the script, see +# https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/contributor-docs/maintainers-guide.md#provision-new-runners # Defaults NAME_SUFFIX="it" @@ -22,6 +23,10 @@ BASE_NAME="gitactions-runner" REPO_NAME="DataflowTemplates" REPO_OWNER="GoogleCloudPlatform" GH_RUNNER_VERSION="2.299.1" + +MACHINE_TYPE="n1-highmem-32" +BOOT_DISK_SIZE="200GB" + VERBOSE=0 ############################################################ @@ -44,6 +49,8 @@ Help() echo "o (optional) Set the owner of the GitHub repo. Default '$REPO_OWNER'" echo "s (optional) Set the number of runners. Default $SIZE" echo "v (optional) Set the gitactions runner version. Default $GH_RUNNER_VERSION" + echo "m (optional) Set the machine type for the GCE VM runner. $MACHINE_TYPE" + echo "b (optional) Set the boot disk size for the GCE VM runner. $BOOT_DISK_SIZE" echo "V Verbose mode." echo "h Print this Help." echo @@ -79,6 +86,10 @@ while getopts ":h:Vp:a:t:n:S:r:o:s:v:" option; do SIZE=$OPTARG;; v) # Enter a version GH_RUNNER_VERSION=$OPTARG;; + m) # Enter a machine type + MACHINE_TYPE=$OPTARG;; + b) # Enter a boot disk size + BOOT_DISK_SIZE=$OPTARG;; V) # Verbose VERBOSE=1;; \?) # Invalid option @@ -163,8 +174,6 @@ gcloud secrets add-iam-policy-binding $SECRET_NAME \ IMAGE_FAMILY="ubuntu-2004-lts" IMAGE_PROJECT="ubuntu-os-cloud" BOOT_DISK_TYPE="pd-balanced" -BOOT_DISK_SIZE="200GB" -MACHINE_TYPE="n1-highmem-16" SCOPE="cloud-platform" if [ $VERBOSE -eq 1 ]; then echo; echo "Creating instance template: $INSTANCE_TEMPLATE_NAME..."; fi if [ $VERBOSE -eq 1 ]; then @@ -181,7 +190,7 @@ gcloud compute instance-templates create $INSTANCE_TEMPLATE_NAME \ --image-project=$IMAGE_PROJECT \ --boot-disk-type=$BOOT_DISK_TYPE \ --boot-disk-size=$BOOT_DISK_SIZE \ - --machine-type="MACHINE_TYPE" \ + --machine-type=$MACHINE_TYPE \ --scopes=$SCOPE \ --service-account=${SA_EMAIL} \ --metadata-from-file=startup-script=startup-script-${NAME_SUFFIX}.sh,shutdown-script=shutdown-script-${NAME_SUFFIX}.sh diff --git a/.github/scripts/startup-script.sh b/.github/scripts/startup-script.sh index 5ee8470b18..fa782dae38 100644 --- a/.github/scripts/startup-script.sh +++ b/.github/scripts/startup-script.sh @@ -22,30 +22,28 @@ ulimit -n 65536 # increase max virtual memory sudo sysctl -w vm.max_map_count=262144 +# update git +sudo add-apt-repository ppa:git-core/ppa -y +sudo apt update +sudo apt install git -y + # install jq -apt-get update -apt-get -y install jq +sudo apt install jq -y # install maven -sudo apt update sudo apt install git maven -y -# update git -sudo add-apt-repository ppa:git-core/ppa -y -sudo apt-get update -sudo apt-get install git -y - # install gh -sudo type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y) -sudo curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ +sudo apt install curl -y \ +&& sudo curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ && sudo apt update \ && sudo apt install gh -y # install docker -sudo apt-get update -sudo apt-get install \ +sudo apt update +sudo apt install \ ca-certificates \ curl \ gnupg \ @@ -55,8 +53,8 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get update -sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y +sudo apt update +sudo apt install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y # add user to docker group sudo groupadd docker diff --git a/.github/workflows/go-pr.yml b/.github/workflows/go-pr.yml index a7b73b1d06..7e1573f6fd 100644 --- a/.github/workflows/go-pr.yml +++ b/.github/workflows/go-pr.yml @@ -35,9 +35,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - name: Setup Go - uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: '1.21' - name: Run Fmt @@ -53,9 +53,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - name: Setup Go - uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: go-version: '1.21' # By nature, this also makes sure that everything builds diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml index 887b3b2ab7..fa91a74b8d 100644 --- a/.github/workflows/java-pr.yml +++ b/.github/workflows/java-pr.yml @@ -27,10 +27,22 @@ on: # This will make it easier to verify action changes don't break anything. - '.github/actions/setup-env/*' - '.github/workflows/java-pr.yml' + # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml) + - '!v2/datastream-to-spanner/**' + - '!v2/spanner-common/**' + - '!v2/spanner-change-streams-to-sharded-file-sink/**' + - '!v2/gcs-to-sourcedb/**' + - '!v2/spanner-migrations-sdk/**' + - '!v2/spanner-custom-shard/**' + - '!v2/sourcedb-to-spanner/**' schedule: - cron: "0 */12 * * *" workflow_dispatch: +concurrency: + group: java-pr-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + env: MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error @@ -44,13 +56,11 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Spotless - run: ./cicd/run-spotless --changed-files="${{ steps.setup-env.outputs.changed-files }}" + run: ./cicd/run-spotless checkstyle_check: name: Checkstyle timeout-minutes: 10 @@ -58,13 +68,11 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Checkstyle - run: ./cicd/run-checkstyle --changed-files="${{ steps.setup-env.outputs.changed-files }}" + run: ./cicd/run-checkstyle java_build: name: Build timeout-minutes: 60 @@ -72,13 +80,11 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Build - run: ./cicd/run-build --changed-files="${{ steps.setup-env.outputs.changed-files }}" + run: ./cicd/run-build - name: Cleanup Java Environment uses: ./.github/actions/cleanup-java-env java_unit_tests: @@ -89,13 +95,11 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Unit Tests - run: ./cicd/run-unit-tests --changed-files="${{ steps.setup-env.outputs.changed-files }}" + run: ./cicd/run-unit-tests - name: Upload Unit Tests Report uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 if: always() # always run even if the previous step fails @@ -120,15 +124,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Integration Smoke Tests run: | ./cicd/run-it-smoke-tests \ - --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ @@ -151,15 +152,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Integration Tests run: | ./cicd/run-it-tests \ - --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ @@ -183,15 +181,12 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - with: - fetch-depth: 2 # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F - name: Setup Environment id: setup-env uses: ./.github/actions/setup-env - name: Run Load Tests run: | ./cicd/run-load-tests \ - --changed-files="${{ steps.setup-env.outputs.changed-files }}" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ diff --git a/.github/workflows/prepare-java-cache.yml b/.github/workflows/prepare-java-cache.yml index 399a3e8bd1..33af88c88b 100644 --- a/.github/workflows/prepare-java-cache.yml +++ b/.github/workflows/prepare-java-cache.yml @@ -58,7 +58,7 @@ jobs: run: | echo "CACHE_KEY=''" >> $GITHUB_ENV - name: Checkout Code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 - name: Setup Java id: setup-java uses: ./.github/actions/setup-java-env diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01dd51cd82..e31a80c1d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ permissions: jobs: release: name: Create Release - runs-on: [self-hosted, it] + runs-on: [self-hosted, release] steps: - name: Get releaser identity run: | @@ -68,13 +68,12 @@ jobs: id: setup-env uses: ./.github/actions/setup-env - name: Run Build - run: ./cicd/run-build --changed-files="pom.xml" + run: ./cicd/run-build - name: Run Unit Tests - run: ./cicd/run-unit-tests --changed-files="pom.xml" + run: ./cicd/run-unit-tests - name: Run Integration Smoke Tests run: | ./cicd/run-it-smoke-tests \ - --changed-files="pom.xml" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ @@ -84,7 +83,6 @@ jobs: - name: Run Integration Tests run: | ./cicd/run-it-tests \ - --changed-files="pom.xml" \ --it-region="us-central1" \ --it-project="cloud-teleport-testing" \ --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ diff --git a/.github/workflows/scorecards-analysis.yml b/.github/workflows/scorecards-analysis.yml index e066e6bbc0..f7cbc7b10e 100644 --- a/.github/workflows/scorecards-analysis.yml +++ b/.github/workflows/scorecards-analysis.yml @@ -22,7 +22,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0 with: persist-credentials: false diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml new file mode 100644 index 0000000000..a6722035ec --- /dev/null +++ b/.github/workflows/spanner-pr.yml @@ -0,0 +1,201 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks that are intended to run on PRs containing Java code. + +name: Spanner PR + +on: + pull_request: + branches: + - 'main' + paths: + # Include spanner paths only + - '.github/workflows/spanner-pr.yml' + - 'v2/datastream-to-spanner/**' + - 'v2/spanner-common/**' + - 'v2/spanner-change-streams-to-sharded-file-sink/**' + - 'v2/gcs-to-sourcedb/**' + - 'v2/spanner-migrations-sdk/**' + - 'v2/spanner-custom-shard/**' + - 'v2/sourcedb-to-spanner/**' + schedule: + - cron: "0 */12 * * *" + workflow_dispatch: + +concurrency: + group: java-pr-${{ github.event.issue.number || github.run_id }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error + +permissions: read-all + +jobs: + spotless_check: + name: Spotless + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Spotless + run: | + ./cicd/run-spotless \ + --modules-to-build="SPANNER" + checkstyle_check: + name: Checkstyle + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Checkstyle + run: | + ./cicd/run-checkstyle \ + --modules-to-build="SPANNER" + java_build: + name: Build + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Build + run: | + ./cicd/run-build \ + --modules-to-build="SPANNER" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_unit_tests: + name: Unit Tests + needs: [java_build] + timeout-minutes: 60 + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Unit Tests + run: | + ./cicd/run-unit-tests \ + --modules-to-build="SPANNER" + - name: Upload Unit Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.0.1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: GoogleCloudPlatform/DataflowTemplates + files: 'target/site/jacoco-aggregate/jacoco.xml' + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_smoke_tests_templates: + name: Dataflow Templates Integration Smoke Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests] + timeout-minutes: 60 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Smoke Tests + run: | + ./cicd/run-it-smoke-tests \ + --modules-to-build="SPANNER" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Smoke Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_integration_tests_templates: + name: Dataflow Templates Integration Tests + needs: [java_integration_smoke_tests_templates] + timeout-minutes: 180 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, it] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --modules-to-build="SPANNER" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env + java_load_tests_templates: + if: contains(github.event.pull_request.labels.*.name, 'run-load-tests') + name: Dataflow Templates Load Tests + needs: [spotless_check, checkstyle_check, java_build, java_unit_tests, java_integration_tests_templates] + timeout-minutes: 600 + # Run on any runner that matches all the specified runs-on values. + runs-on: [self-hosted, perf] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Load Tests + run: | + ./cicd/run-load-tests \ + --modules-to-build="SPANNER" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..97f6446e57 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,30 @@ +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '23 2 * * *' + workflow_dispatch: + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has been marked as stale due to 180 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the issue at any time. Thank you for your contributions.' + close-issue-message: 'This issue has been closed due to lack of activity. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time.' + stale-pr-message: 'This pull request has been marked as stale due to 180 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time. Thank you for your contributions.' + close-pr-message: 'This pull request has been closed due to lack of activity. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time.' + days-before-stale: 180 + operations-per-run: 100 diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index faf965f00f..899d997619 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -18,53 +18,33 @@ package flags import ( "flag" - "fmt" - "log" - "regexp" "strings" ) +const ( + ALL = "ALL" + SPANNER = "SPANNER" +) + // Avoid making these vars public. var ( - changedFiles string + modulesToBuild string + moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,plugins/templates-maven-plugin"} ) // Registers all common flags. Must be called before flag.Parse(). func RegisterCommonFlags() { - flag.StringVar(&changedFiles, "changed-files", "", "List of changed files as a comma-separated string") + flag.StringVar(&modulesToBuild, "modules-to-build", ALL, "List of modules to build/run commands against") } -// Returns all changed files with regexes. If no regexes are passed, all files are returned. If even one -// is passed, then only file paths with a match anywhere in the file will be returned. If multiple are -// passed, it is equivalent to (regex1|regex2|...|regexN) -func ChangedFiles(regexes ...string) []string { - if len(changedFiles) == 0 { - log.Println("WARNING: No changed files were passed. This could indicate an error.") - return make([]string, 0) - } - - files := strings.Split(changedFiles, ",") - if len(regexes) == 0 { - return files +// Returns all modules to build. +func ModulesToBuild() []string { + m := modulesToBuild + if val, ok := moduleMap[modulesToBuild]; ok { + m = val } - - var fullRegex string - if len(regexes) == 1 { - fullRegex = regexes[0] - } else { - fullRegex = fmt.Sprintf("(%s)", strings.Join(regexes, "|")) - } - re := regexp.MustCompile(fullRegex) - - results := make([]string, 0) - for _, f := range files { - if re.MatchString(f) { - results = append(results, f) - } - } - - if len(results) == 0 { - log.Println("INFO: All changed files got filtered out.") + if len(m) == 0 { + return make([]string, 0) } - return results + return strings.Split(m, ",") } diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 0881b1bb21..fc49b542b3 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -21,79 +21,34 @@ import ( "testing" ) -func TestChangedFilesNoRegex(t *testing.T) { +func TestModulesToBuild(t *testing.T) { tests := []struct { input string expected []string }{ { - input: "file1,file2", - expected: []string{"file1", "file2"}, + input: "m1,m2", + expected: []string{"m1", "m2"}, }, { - input: "file1", - expected: []string{"file1"}, - }, - } - - for _, test := range tests { - changedFiles = test.input - actual := ChangedFiles() - if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Returned files are not equal. Expected %v. Got %v.", test.expected, actual) - } - } -} - -func TestChangedFilesNoRegexEmpty(t *testing.T) { - changedFiles = "" - actual := ChangedFiles() - if len(actual) != 0 { - t.Errorf("Expected empty slice, but got %v of len %v", actual, len(actual)) - } -} - -func TestChangedFilesRegexes(t *testing.T) { - tests := []struct { - files string - regexes []string - expected []string - }{ - { - files: "file1,file2,file3", - regexes: []string{"file[1|3]"}, - expected: []string{"file1", "file3"}, + input: "m1", + expected: []string{"m1"}, }, { - files: "file1,file2,file3", - regexes: []string{"f.+1", "fi.+3"}, - expected: []string{"file1", "file3"}, + input: "ALL", + expected: []string{}, }, { - files: "file1,file2,fileN", - regexes: []string{"\\d"}, - expected: []string{"file1", "file2"}, - }, - { - files: "foo.c,bar.cc", - regexes: []string{"\\.c$"}, - expected: []string{"foo.c"}, + input: "SPANNER", + expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "plugins/templates-maven-plugin"}, }, } for _, test := range tests { - changedFiles = test.files - actual := ChangedFiles(test.regexes...) + modulesToBuild = test.input + actual := ModulesToBuild() if !reflect.DeepEqual(actual, test.expected) { - t.Errorf("Returned files are not equal. Expected %v. Got %v.", test.expected, actual) + t.Errorf("Returned modules are not equal. Expected %v. Got %v.", test.expected, actual) } } } - -func TestChangedFilesRegexesNoMatch(t *testing.T) { - changedFiles = "foo,bar" - actual := ChangedFiles("file") - if len(actual) != 0 { - t.Errorf("Expected empty slice but got %v", actual) - } -} diff --git a/cicd/internal/op/maven.go b/cicd/internal/op/maven.go index 74cee339cf..93c659a04b 100644 --- a/cicd/internal/op/maven.go +++ b/cicd/internal/op/maven.go @@ -18,6 +18,8 @@ package op import ( "strings" + + "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/flags" ) // Runs the given Maven command on a specified POM file. Considering the input, this is equivalent to: @@ -29,15 +31,18 @@ func RunMavenOnPom(pom string, cmd string, args ...string) error { fullArgs = append(fullArgs, "-f", pom) fullArgs = append(fullArgs, "-e") fullArgs = append(fullArgs, args...) - + modules := flags.ModulesToBuild() + if len(modules) != 0 { + moduleArgs := []string{"-pl", strings.Join(modules, ",")} + fullArgs = append(fullArgs, moduleArgs...) + fullArgs = append(fullArgs, "-am") + } return RunCmdAndStreamOutput("mvn", fullArgs) } // Runs the given Maven command on a specified module. Considering the input, this is equivalent to: // // mvn -B {cmd} -f {pom} -pl {module} {args...} -func RunMavenOnModule(pom string, cmd string, module string, args ...string) error { - // fullArgs := []string{"-pl", module} - // fullArgs = append(fullArgs, args...) +func RunMavenOnModule(pom string, cmd string, args ...string) error { return RunMavenOnPom(pom, cmd, args...) } diff --git a/cicd/internal/workflows/maven-workflows.go b/cicd/internal/workflows/maven-workflows.go index 74ecb6d7c2..75eecc083e 100644 --- a/cicd/internal/workflows/maven-workflows.go +++ b/cicd/internal/workflows/maven-workflows.go @@ -17,15 +17,8 @@ package workflows import ( - "fmt" - "log" - "path/filepath" - "strconv" - "strings" - - "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/flags" "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/op" - "github.com/GoogleCloudPlatform/DataflowTemplates/cicd/internal/repo" + "strconv" ) const ( @@ -37,12 +30,6 @@ const ( spotlessCheckCmd = "spotless:check" checkstyleCheckCmd = "checkstyle:check" - // regexes - javaFileRegex = "\\.java$" - xmlFileRegex = "\\.xml$" - markdownFileRegex = "\\.md$" - pomFileRegex = "pom\\.xml$" - // notable files unifiedPom = "pom.xml" ) @@ -202,80 +189,13 @@ func (*mvnVerifyWorkflow) Run(args ...string) error { } func RunForChangedModules(cmd string, args ...string) error { - changed := flags.ChangedFiles(javaFileRegex, xmlFileRegex) - if len(changed) == 0 { - return nil - } - parsedArgs := []string{} for _, arg := range args { if arg != "" { parsedArgs = append(parsedArgs, arg) } } - - // Collect the modules together for a single call. Maven can work out the install order. - modules := make([]string, 0) - - // We need to append the base dependency modules, because they are needed to build all - // other modules. - for root, children := range repo.GetModulesForPaths(changed) { - if len(children) == 0 { - modules = append(modules, root) - continue - } - - // A change to the root POM could impact all children, so build them all. - buildAll := false - for _, c := range changed { - if c == filepath.Join(root, "pom.xml") { - buildAll = true - break - } - } - if buildAll { - modules = append(modules, root) - continue - } - - withoutRoot := removeRoot(children) - if len(withoutRoot) == 0 { - log.Printf("All files under %s were irrelevant root-level files", root) - } - for _, m := range withoutRoot { - modules = append(modules, fmt.Sprintf("%s/%s", root, m)) - } - } - - if len(modules) == 0 { - log.Println("All modules were filtered out.") - return nil - } - - has_it := false - has_common := false - has_v2 := false - for _, module := range modules { - if len(module) > 1 && module[:2] == "it" { - has_it = true - } - if module == "v2/common" { - has_common = true - } - if module == "v2" { - has_v2 = true - } - } - if has_it && !has_common { - modules = append(modules, "v2/common") - } - if (has_v2 || has_common) && !has_it { - modules = append(modules, "it") - } - - modules = append(modules, "plugins/templates-maven-plugin") - - return op.RunMavenOnModule(unifiedPom, cmd, strings.Join(modules, ","), parsedArgs...) + return op.RunMavenOnModule(unifiedPom, cmd, parsedArgs...) } type spotlessCheckWorkflow struct{} diff --git a/tutorials/flex-template.md b/contributor-docs/add-flex-template.md similarity index 99% rename from tutorials/flex-template.md rename to contributor-docs/add-flex-template.md index 8646ca5ef0..ab77d1f8e3 100644 --- a/tutorials/flex-template.md +++ b/contributor-docs/add-flex-template.md @@ -1,4 +1,4 @@ -# Flex Template Tutorial +# Adding a Flex Template ## Overview diff --git a/tutorials/load-test.md b/contributor-docs/add-integration-or-load-test.md similarity index 86% rename from tutorials/load-test.md rename to contributor-docs/add-integration-or-load-test.md index 6b04e7f275..ef56147eb5 100644 --- a/tutorials/load-test.md +++ b/contributor-docs/add-integration-or-load-test.md @@ -1,4 +1,4 @@ -# Load Test Tutorial +# Adding an Integration or Load Test ## Overview @@ -305,6 +305,8 @@ synthetic records/messages at a user specified QPS. These messages are generated based on a user specified schema. Please look at [this](https://github.com/GoogleCloudPlatform/DataflowTemplates/tree/main/v2/streaming-data-generator#creating-the-schema-file) guide to create a schema file. +Data generators are typically used in load tests, not integration tests. + Pre-existing schema templates can also be used instead of specifying a schema. Supported schema templates are, @@ -373,6 +375,32 @@ of messages. If messageLimit is not specified, the data generator generates messages at specified QPS till timeout. +## Write an Integration test + +Integration tests will be written using JUnit. The structure of the load test will +vary on whether the pipeline under test is a `Batch` or `Streaming` pipeline and +the type of test. + +### Structure +First extend the test class from the [TemplateTestBase](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/TemplateTestBase.java) +class. TemplateTestBase contains helper methods which abstract irrelevant +information and make it easier to write tests. It also defines some +clients and variables which are useful for writing tests. + +```java +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import org.apache.beam.it.gcp.TemplateTestBase; +import org.junit.runners.JUnit4; +@Category({TemplateIntegrationTest.class}) +@TemplateIntegrationTest(WordCount.class) +@RunWith(JUnit4.class) +public class WordCountIT extends TemplateTestBase { + +} +``` + +From there, you can add test cases as described below in [Test Cases](#test-cases). + ## Write a Load test Load tests will be written using JUnit. The structure of the load test will @@ -389,8 +417,13 @@ NOTE: Any class extending `LoadTestBase` will need to implement a `launcher` method which creates the appropriate PipelineLauncher to be used for the test. ```java -import com.google.cloud.teleport.it.LoadTestBase; -public class WordCountLoadTest extends LoadTestBase { +import com.google.cloud.teleport.metadata.TemplateLoadTest; +import com.google.cloud.teleport.it.TemplateLoadTestBase; +import org.junit.runners.JUnit4; +@Category(TemplateLoadTest.class) +@TemplateLoadTest(WordCount.class) +@RunWith(JUnit4.class) +public class WordCountLoadTest extends TemplateLoadTestBase { PipelineLauncher launcher() { return new DefaultPipelineLauncher.builder().setCredentials(CREDENTIALS).build(); @@ -403,7 +436,13 @@ public class WordCountLoadTest extends LoadTestBase { NOTE: For Google-provided template load tests, `TemplateLoadTestBase` can be used, whereas for Apache Beam I/O load tests `IOLoadTestBase` can be used. -### Test cases +From there, you can add test cases as described below in [Test Cases](#test-cases). + +## Test cases + +There are generally 2 classes of load and integration tests: backlog tests and steady state +tests (streaming-only). These largely function in the same manner, with minor differences +which are called out in the code below. #### Backlog Tests @@ -423,6 +462,9 @@ public void testBacklog() { TableId table = bigQueryResourceManager.createTable(testName, SCHEMA); // Generate fake data to Pub/Sub topic + // In a normal integration test (small amount of data), you can use the resource manager + // directly. For example: + // pubsubResourceManager.publish(topic, ImmutableMap.of(), messageData); DataGenerator dataGenerator = DataGenerator.builderWithSchemaTemplate(testName + "-data-generator", "GAME_EVENT") .setQPS("200000") @@ -437,7 +479,9 @@ public void testBacklog() { .addParameter("inputSubscription", inputSubscription.toString()) .addParameter("outputTableSpec", toTableSpec(project, table)) .build(); - // launch pipeline under test + // Launch pipeline under test. For load tests, use the pipelineLauncher + // For integration tests, you can use launchTemplate, for example: + // LaunchInfo info = launchTemplate(options) LaunchInfo info = pipelineLauncher.launch(PROJECT, REGION, options); assertThatPipeline(info).isRunning(); @@ -469,6 +513,9 @@ public void testSteadyState1hr() { TableId table = bigQueryResourceManager.createTable( testName, SCHEMA, System.currentTimeMillis() + 7200000); // expire in 2 hrs // Generate fake data to Pub/Sub topic at 100,000 QPS + // In a normal integration test (small amount of data), you can use the resource manager + // directly. For example: + // pubsubResourceManager.publish(topic, ImmutableMap.of(), messageData); DataGenerator dataGenerator = DataGenerator.builderWithSchemaTemplate(testName + "-data-generator","GAME_EVENT") .setQPS("100000") @@ -481,7 +528,9 @@ public void testSteadyState1hr() { .addParameter("inputSubscription", inputSubscription.toString()) .addParameter("outputTableSpec", toTableSpec(project, table)) .build(); - // launch pipeline under test + // Launch pipeline under test. For load tests, use the pipelineLauncher + // For integration tests, you can use launchTemplate, for example: + // LaunchInfo info = launchTemplate(options) LaunchInfo info = pipelineLauncher.launch(project, region, options); assertThatPipeline(info).isRunning(); @@ -543,3 +592,14 @@ For manually running a load test execute the following commands on the CLI use t Additional parameters can be specified using `-D=` +To run an integration test, follow step 1 above, then run the specific test: + +```shell +mvn clean verify \ + -PtemplatesIntegrationTests \ + -Dtest="#" -Dproject=$PROJECT \ + -DartifactBucket=$ARTIFACT_BUCKET -DexportProject=$EXPORT_PROJECT \ + -DexportDataset=$EXPORT_DATASET -DexportTable=$EXPORT_TABLE \ + -DfailIfNoTests=false -DtrimStackTrace=false +``` + diff --git a/contributor-docs/code-contributions.md b/contributor-docs/code-contributions.md index d22e6b6596..4ce6d2201e 100644 --- a/contributor-docs/code-contributions.md +++ b/contributor-docs/code-contributions.md @@ -226,6 +226,8 @@ Notes: ### Running Integration Tests +For information on adding integration tests, see [Adding an Integration or Load Test](./add-integration-or-load-test.md) + To run integration tests, the developer plugin can be also used to stage template on-demand (in case the parameter `-DspecPath=` is not specified). For example, to run all the integration tests in a specific module (in the example below, `v2/googlecloud-to-googlecloud`): @@ -243,6 +245,30 @@ The parameter `-Dtest=` can be given to test a single class (e.g., `-Dtest=Pubsu The same happens when the test is executed from an IDE, just make sure to add the parameters `-Dproject=`, `-DartifactBucket=` and `-Dregion=` as program or VM arguments. +### Running Load Tests + +For information on adding and running load tests, see [Adding a Load Test](./add-integration-or-load-test.md). + +### Adding New Templates + +If you are interested in introducing a new template, please file an issue using the [Google Issue Tracker](https://issuetracker.google.com/issues/new?component=187168&template=0) before doing so. You need approval before moving forward. Any new templates must be flex templates in the v2 directory. + +Once you have approval, follow these steps from the root of the repo to get started on your template. + +First, install the [maven plugin](#templates-plugin), then create a subdirectory in the repository: `v2/`. `` should follow the pattern `-to-`. From there, you can follow the steps in [Adding a Flex Template](./add-flex-template.md) to develop your template. + +All new templates must comply with the following guidance: + +- [ ] Template addition has been approved by the Dataflow team in a [Google Issue Tracker Issue](https://issuetracker.google.com/issues/new?component=187168&template=0). +- [ ] The template must be a Flex Template located in the `v2/