Merge branch 'GoogleCloudPlatform:main' into rajkumargupta-dataflowTe…

…mplates

.github/actions/setup-env/action.yml

@@ -46,23 +46,15 @@ inputs:
     description: 'The version of Python to install'
     required: false
     default: '3.11'
-outputs:
-  changed-files:
-    description: 'Comma-separated list of files that were changed'
-    value: ${{ steps.changed-files.outputs.all_changed_and_modified_files }}
 
 runs:
   using: 'composite'
   steps:
     - name: Setup Go
-      uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
       with:
         go-version: ${{ inputs.go-version }}
-    - name: Get Changed Files
-      id: changed-files
-      uses: tj-actions/changed-files@61ee456a9d0f512e7ecfdf28863634c97dae2d16
-      with:
-        separator: ','
+        cache: false
     # It shouldn't take too long to build all of this, and it will at least
     # make running the target program easier
     - name: Build CI/CD

.github/actions/setup-java-env/action.yml

@@ -40,7 +40,7 @@ runs:
   using: 'composite'
   steps:
     - name: Setup Java
-      uses: actions/setup-java@a12e082d834968c1847f782019214fadd20719f6
+      uses: actions/setup-java@5896cecc08fd8a1fbdfaf517e29b571164b031f7 # v4.2.0
       with:
         distribution: 'temurin'
         java-version: ${{ inputs.java-version }}
@@ -57,7 +57,7 @@ runs:
           echo "YESTERDAY=$KEY" >> $GITHUB_ENV
         fi
     - name: Setup Cache
-      uses: actions/cache@72d1e4fdff0ff7b1b6e86b415f2d4f5941e5c006
+      uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
       id: setup-cache
       with:
         path: |

.github/scripts/configure-runners.sh

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# For running the script, see go/templates-gitactions-script
+# For running the script, see
+# https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/contributor-docs/maintainers-guide.md#provision-new-runners
 
 # Defaults
 NAME_SUFFIX="it"
@@ -22,6 +23,10 @@ BASE_NAME="gitactions-runner"
 REPO_NAME="DataflowTemplates"
 REPO_OWNER="GoogleCloudPlatform"
 GH_RUNNER_VERSION="2.299.1"
+
+MACHINE_TYPE="n1-highmem-32"
+BOOT_DISK_SIZE="200GB"
+
 VERBOSE=0
 
 ############################################################
@@ -44,6 +49,8 @@ Help()
    echo "o     (optional) Set the owner of the GitHub repo. Default '$REPO_OWNER'"
    echo "s     (optional) Set the number of runners. Default $SIZE"
    echo "v     (optional) Set the gitactions runner version. Default $GH_RUNNER_VERSION"
+   echo "m     (optional) Set the machine type for the GCE VM runner. $MACHINE_TYPE"
+   echo "b     (optional) Set the boot disk size for the GCE VM runner. $BOOT_DISK_SIZE"
    echo "V     Verbose mode."
    echo "h     Print this Help."
    echo
@@ -79,6 +86,10 @@ while getopts ":h:Vp:a:t:n:S:r:o:s:v:" option; do
          SIZE=$OPTARG;;
       v) # Enter a version
          GH_RUNNER_VERSION=$OPTARG;;
+      m) # Enter a machine type
+         MACHINE_TYPE=$OPTARG;;
+      b) # Enter a boot disk size
+         BOOT_DISK_SIZE=$OPTARG;;
       V) # Verbose
          VERBOSE=1;;
       \?) # Invalid option
@@ -163,8 +174,6 @@ gcloud secrets add-iam-policy-binding $SECRET_NAME \
 IMAGE_FAMILY="ubuntu-2004-lts"
 IMAGE_PROJECT="ubuntu-os-cloud"
 BOOT_DISK_TYPE="pd-balanced"
-BOOT_DISK_SIZE="200GB"
-MACHINE_TYPE="n1-highmem-16"
 SCOPE="cloud-platform"
 if [ $VERBOSE -eq 1 ]; then echo; echo "Creating instance template: $INSTANCE_TEMPLATE_NAME..."; fi
 if [ $VERBOSE -eq 1 ]; then
@@ -181,7 +190,7 @@ gcloud compute instance-templates create $INSTANCE_TEMPLATE_NAME \
   --image-project=$IMAGE_PROJECT \
   --boot-disk-type=$BOOT_DISK_TYPE \
   --boot-disk-size=$BOOT_DISK_SIZE \
-  --machine-type="MACHINE_TYPE" \
+  --machine-type=$MACHINE_TYPE \
   --scopes=$SCOPE \
   --service-account=${SA_EMAIL} \
   --metadata-from-file=startup-script=startup-script-${NAME_SUFFIX}.sh,shutdown-script=shutdown-script-${NAME_SUFFIX}.sh

.github/scripts/startup-script.sh

@@ -22,30 +22,28 @@ ulimit -n 65536
 # increase max virtual memory
 sudo sysctl -w vm.max_map_count=262144
 
+# update git
+sudo add-apt-repository ppa:git-core/ppa -y
+sudo apt update
+sudo apt install git -y
+
 # install jq
-apt-get update
-apt-get -y install jq
+sudo apt install jq -y
 
 # install maven
-sudo apt update
 sudo apt install git maven -y
 
-# update git
-sudo add-apt-repository ppa:git-core/ppa -y
-sudo apt-get update
-sudo apt-get install git -y
-
 # install gh
-sudo type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y)
-sudo curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+sudo apt install curl -y \
+&& sudo curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
 && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
 && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
 && sudo apt update \
 && sudo apt install gh -y
 
 # install docker
-sudo apt-get update
-sudo apt-get install \
+sudo apt update
+sudo apt install \
     ca-certificates \
     curl \
     gnupg \
@@ -55,8 +53,8 @@ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o
 echo \
   "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
   $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-sudo apt-get update
-sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y
+sudo apt update
+sudo apt install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y
 
 # add user to docker group
 sudo groupadd docker

.github/workflows/go-pr.yml

@@ -35,9 +35,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
       - name: Setup Go
-        uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version: '1.21'
       - name: Run Fmt
@@ -53,9 +53,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
       - name: Setup Go
-        uses: actions/setup-go@44e221478fc6847752e5c574fc7a7b3247b00fbf
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version: '1.21'
       # By nature, this also makes sure that everything builds

.github/workflows/java-pr.yml

@@ -27,10 +27,22 @@ on:
       # This will make it easier to verify action changes don't break anything.
       - '.github/actions/setup-env/*'
       - '.github/workflows/java-pr.yml'
+      # Exclude spanner paths from global run (covered in https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/.github/workflows/spanner-pr.yml)
+      - '!v2/datastream-to-spanner/**'
+      - '!v2/spanner-common/**'
+      - '!v2/spanner-change-streams-to-sharded-file-sink/**'
+      - '!v2/gcs-to-sourcedb/**'
+      - '!v2/spanner-migrations-sdk/**'
+      - '!v2/spanner-custom-shard/**'
+      - '!v2/sourcedb-to-spanner/**'
   schedule:
     - cron: "0 */12 * * *"
   workflow_dispatch:
 
+concurrency:
+  group: java-pr-${{ github.event.issue.number || github.run_id }}
+  cancel-in-progress: true
+
 env:
   MAVEN_OPTS: -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=error
 
@@ -44,41 +56,35 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Spotless
-        run: ./cicd/run-spotless --changed-files="${{ steps.setup-env.outputs.changed-files }}"
+        run: ./cicd/run-spotless
   checkstyle_check:
     name: Checkstyle
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Checkstyle
-        run: ./cicd/run-checkstyle --changed-files="${{ steps.setup-env.outputs.changed-files }}"
+        run: ./cicd/run-checkstyle
   java_build:
     name: Build
     timeout-minutes: 60
     runs-on: [self-hosted, it]
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Build
-        run: ./cicd/run-build --changed-files="${{ steps.setup-env.outputs.changed-files }}"
+        run: ./cicd/run-build
       - name: Cleanup Java Environment
         uses: ./.github/actions/cleanup-java-env
   java_unit_tests:
@@ -89,13 +95,11 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Unit Tests
-        run: ./cicd/run-unit-tests --changed-files="${{ steps.setup-env.outputs.changed-files }}"
+        run: ./cicd/run-unit-tests
       - name: Upload Unit Tests Report
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         if: always() # always run even if the previous step fails
@@ -120,15 +124,12 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Integration Smoke Tests
         run: | 
           ./cicd/run-it-smoke-tests \
-          --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
           --it-region="us-central1" \
           --it-project="cloud-teleport-testing" \
           --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
@@ -151,15 +152,12 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-        with:
-          fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
       - name: Setup Environment
         id: setup-env
         uses: ./.github/actions/setup-env
       - name: Run Integration Tests
         run: | 
           ./cicd/run-it-tests \
-          --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
           --it-region="us-central1" \
           --it-project="cloud-teleport-testing" \
           --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
@@ -183,15 +181,12 @@ jobs:
     steps:
     - name: Checkout Code
       uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      with:
-        fetch-depth: 2  # setup-env action assumes fetch depth of at least 2 for https://github.com/tj-actions/changed-files?tab=readme-ov-file#on-push-%EF%B8%8F
     - name: Setup Environment
       id: setup-env
       uses: ./.github/actions/setup-env
     - name: Run Load Tests
       run: |
         ./cicd/run-load-tests \
-        --changed-files="${{ steps.setup-env.outputs.changed-files }}" \
         --it-region="us-central1" \
         --it-project="cloud-teleport-testing" \
         --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \

.github/workflows/prepare-java-cache.yml

@@ -58,7 +58,7 @@ jobs:
         run: |
           echo "CACHE_KEY=''" >> $GITHUB_ENV
       - name: Checkout Code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
       - name: Setup Java
         id: setup-java
         uses: ./.github/actions/setup-java-env

.github/workflows/release.yml

@@ -23,7 +23,7 @@ permissions:
 jobs:
   release:
     name: Create Release
-    runs-on: [self-hosted, it]
+    runs-on: [self-hosted, release]
     steps:
     - name: Get releaser identity
       run: |
@@ -68,13 +68,12 @@ jobs:
       id: setup-env
       uses: ./.github/actions/setup-env
     - name: Run Build
-      run: ./cicd/run-build --changed-files="pom.xml"
+      run: ./cicd/run-build
     - name: Run Unit Tests
-      run: ./cicd/run-unit-tests --changed-files="pom.xml"
+      run: ./cicd/run-unit-tests
     - name: Run Integration Smoke Tests
       run: |
         ./cicd/run-it-smoke-tests \
-        --changed-files="pom.xml" \
         --it-region="us-central1" \
         --it-project="cloud-teleport-testing" \
         --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
@@ -84,7 +83,6 @@ jobs:
     - name: Run Integration Tests
       run: |
         ./cicd/run-it-tests \
-        --changed-files="pom.xml" \
         --it-region="us-central1" \
         --it-project="cloud-teleport-testing" \
         --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \

.github/workflows/scorecards-analysis.yml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
         with:
           persist-credentials: false