Merge branch 'master' of https://github.com/apache/beam into tox_2_again

.asf.yaml

@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.52.0: {}
     release-2.51.0: {}
     release-2.50.0: {}
     release-2.49.0: {}

.github/REVIEWERS.yml

@@ -40,6 +40,7 @@ labels:
       - kennknowles
       - robertwb
       - bvolpato
+      - m-trieu
     exclusionList: []
   - name: IO
     reviewers:
@@ -52,6 +53,10 @@ labels:
   - name: spanner
     reviewers:
       - nielm
+  - name: bigtable
+    reviewers:
+      - igorbernstein2
+      - mutianf
     exclusionList: []
   - name: Build
     reviewers:

.github/actions/common-rc-validation/action.yaml

@@ -36,19 +36,19 @@ runs:
       shell: bash
       run: |
           echo "---------------------Downloading Python Staging RC----------------------------"
-          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
-          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
-          if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then
+          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+          if [[ ! -f apache-beam-$RELEASE_VER.tar.gz ]]; then
             { echo "Fail to download Python Staging RC files." ;exit 1; }
           fi
 
           echo "--------------------------Verifying Hashes------------------------------------"
-          sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+          sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
 
           `which pip` install --upgrade pip
           `which pip` install --upgrade setuptools
 
     - name: Installing python SDK
       shell: bash
-      run: pip install apache-beam-${RELEASE_VER}.zip[gcp]
+      run: pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]
 

.github/actions/gradle-command-self-hosted-action/action.yml

@@ -24,12 +24,6 @@ inputs:
     required: false
     description: 'Gradle options'
     default: ''
-  default-arguments:
-    required: false
-    description: 'Default gradle switches' # Copied from CommonJobProperties.groovy'
-    default: |
-      --continue -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g \
-      -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses
   max-workers:
     required: false
     description: 'Max number of workers'
@@ -47,5 +41,6 @@ runs:
         if [ -f ~/.m2/settings.xml ]; then
           rm ~/.m2/settings.xml
         fi
-        ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} ${{ inputs.arguments }} \
-        ${{ inputs.default-arguments }}
+        ./gradlew ${{ inputs.gradle-command }} --max-workers=${{ inputs.max-workers }} --continue \
+         -Dorg.gradle.jvmargs=-Xms2g -Dorg.gradle.jvmargs=-Xmx6g -Dorg.gradle.vfs.watch=false -Pdocker-pull-licenses \
+        ${{ inputs.arguments }}

.github/actions/setup-action/action.yml

@@ -69,6 +69,4 @@ runs:
     - name: expose gcloud path
       shell: bash
       run: |
-        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
-    - name: Setup environment
-      uses: ./.github/actions/setup-environment-action
+        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV

.github/actions/setup-environment-action/action.yml

@@ -30,6 +30,10 @@ inputs:
     required: false
     description: 'Install Go version'
     default: ''
+  disable-cache:
+    required: false
+    description: 'Whether to disable the gradle cache'
+    default: false
 
 runs:
   using: "composite"
@@ -48,7 +52,7 @@ runs:
     - name: Setup Gradle
       uses: gradle/gradle-build-action@v2
       with:
-        cache-read-only: false
+        cache-read-only: ${{ inputs.disable-cache }}
     - name: Install Go
       if: ${{ inputs.go-version != '' }}
       uses: actions/setup-go@v3

.github/autolabeler.yml

@@ -31,6 +31,7 @@ python: ["sdks/python/**/*", "learning/katas/python/**/*"]
 typescript: ["sdks/typescript/**/*"]
 vendor: ["vendor/**/*"]
 website: ["website/**/*"]
+yaml: ["sdks/python/apache_beam/yaml/**"]
 
 # Extensions
 extensions: ["sdks/java/extensions/**/*", "runners/extensions-java/**/*"]
@@ -68,6 +69,7 @@ io:  ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/
 "redis":  ["sdks/java/io/redis/**/*"]
 "solr":  ["sdks/java/io/solr/**/*"]
 "spanner": ["sdks/go/pkg/beam/io/spannerio/**/*", "sdks/python/apache_beam/io/gcp/spanner.py", "sdks/python/apache_beam/io/gcp/experimental/spannerio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/**/*"]
+"bigtable": ["sdks/go/pkg/beam/io/bigtableio/**/*", "sdks/go/pkg/beam/io/xlang/bigtableio/**/*", "sdks/python/apache_beam/io/gcp/bigtableio.py", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/**/*"]
 "synthetic":  ["sdks/java/io/synthetic/**/*"]
 "tests":  ["sdks/java/io/file-based-io-tests/**/*"]
 "thrift": ["sdks/java/io/thrift/**/*"]

.github/build.gradle

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+buildscript {
+  repositories {
+    mavenCentral()
+  }
+  dependencies {
+    classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2'
+  }
+}
+
+/** check that yml are valid */
+task check {
+  doLast {
+    fileTree("${project.projectDir}/workflows").matching {
+      include "*.yml"
+      include "*.yaml"
+    }.each {
+      def fname = it.getName()
+      // attempt load yml to make sure its valid
+      def workflow = new org.yaml.snakeyaml.Yaml().load(it.newInputStream())
+
+      // additional guards for running all tests functionality
+      // TODO(yathu) expand this also to post commits prior teardown Jenkins postcommits
+      if ( fname.startsWith("beam_PreCommit")) {
+        List paths
+        try {
+          paths = workflow.getAt(true).pull_request_target.paths as List
+        } catch (Exception e) {
+          throw new GradleException("Fail to get the trigger path for ${fname}. " +
+                  "Make sure precommit has a pull_request_target trigger.", e)
+        }
+        // precommit should triggered by this specific file
+        if (paths != null && !paths.contains('release/trigger_all_tests.json')) {
+          throw new GradleException("Error validating ${fname}: " +
+                  "Please add 'release/trigger_all_tests.json' to the trigger path for release verification run properly")
+        }
+      }
+    }
+  }
+}
+
+task preCommit {
+  dependsOn check
+}

.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl

@@ -27,7 +27,7 @@ spec:
     name: ${name}
   minReplicas: ${min_runners}
   maxReplicas: ${max_runners}
-  %{~ if webhook_scaling == "true" ~}
+  %{~ if webhook_scaling ~}
   scaleUpTriggers:
     - githubEvent:
         workflowJob: {}

.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl

@@ -22,7 +22,11 @@ metadata:
   name: ${name}
 spec:
   template:
+    metadata:
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
     spec:
+      dockerMTU: 1460
       %{~ if selector == true  ~}
       nodeSelector:
         runner-pool: ${name} 

.github/gh-actions-self-hosted-runners/arc/environments/beam.env

@@ -20,24 +20,28 @@
 project_id = "apache-beam-testing"
 region = "us-central1"
 zone = "us-central1-b"
-environment = "beam"
+environment = "beam-prod"
 ingress_domain = "action.beam.apache.org" 
 organization = "apache"
 repository = "beam"
 github_app_id_secret_name = "gh-app_id"
 github_app_install_id_secret_name = "gh-app_installation_id"
 github_private_key_secret_name = "gh-pem_key"
 deploy_webhook = "true"
+existing_vpc_name = "default"
+existing_ip_name = "beam-arc-webhook-ip"
+subnetwork_cidr_range = "10.119.0.0/20"
+service_account_id = "[email protected]"
 runner_group = "beam"
 main_runner = {
     name = "main-runner"
     runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
     machine_type = "e2-standard-16"
     min_node_count = "1"
-    max_node_count = "24"
+    max_node_count = "30"
     min_replicas = "1"
-    max_replicas = "200"
-    webhook_scaling = true
+    max_replicas = "240"
+    webhook_scaling = false
     disk_size_gb = 200
     requests = {
         cpu = "2"
@@ -49,10 +53,10 @@ additional_runner_pools = [{
     machine_type = "e2-standard-2"
     runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
     min_node_count = "1"
-    max_node_count = "10"
+    max_node_count = "15"
     min_replicas = "1"
-    max_replicas = "10"
-    webhook_scaling = "true"
+    max_replicas = "15"
+    webhook_scaling = false
     requests = {
         cpu = "1500m"
         memory = "5Gi"
@@ -66,10 +70,10 @@ additional_runner_pools = [{
     machine_type = "c3-highmem-8"
     runner_image = "us-central1-docker.pkg.dev/apache-beam-testing/beam-github-actions/beam-arc-runner:2b20e26bb3b99d8e4f41a3d1d9d2e7080043de5c"
     min_node_count = "1"
-    max_node_count = "10"
+    max_node_count = "15"
     min_replicas = "1"
-    max_replicas = "10"
-    webhook_scaling = "true"
+    max_replicas = "15"
+    webhook_scaling = false
     requests = {
         cpu = "7.5"
         memory = "5Gi"

.github/gh-actions-self-hosted-runners/arc/gke.tf

@@ -21,7 +21,7 @@ resource "google_container_cluster" "actions-runner-gke" {
   project                    = var.project_id
   location                   = var.zone
   initial_node_count         = 1
-  network                    = google_compute_network.actions-runner-network.id
+  network                    = data.google_compute_network.actions-runner-network.id
   subnetwork                 = google_compute_subnetwork.actions-runner-subnetwork.id
   remove_default_node_pool   = true
 
@@ -45,6 +45,7 @@ resource "google_container_node_pool" "main-actions-runner-pool" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+    service_account = data.google_service_account.service_account.email
     tags = ["actions-runner-pool"]
    }
 }
@@ -72,6 +73,7 @@ resource "google_container_node_pool" "additional_runner_pools" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+    service_account = data.google_service_account.service_account.email
     tags = ["actions-runner-pool"]
     labels = {
       "runner-pool" = each.value.name
@@ -90,5 +92,15 @@ resource "google_container_node_pool" "additional_runner_pools" {
 
 
 resource "google_compute_global_address" "actions-runner-ip" {
-  name      = "${var.environment}-actions-runner-ip"
+  count = var.deploy_webhook == "true" && var.existing_ip_name == "" ? 1 : 0
+  name  = "${var.environment}-actions-runner-ip"
+}
+
+data "google_compute_global_address" "actions-runner-ip" {
+  count = var.deploy_webhook == "true" ? 1 : 0
+  name  = var.existing_ip_name == "" ? google_compute_global_address.actions-runner-ip[0].name : var.existing_ip_name
+}
+
+data google_service_account "service_account" {
+  account_id = var.service_account_id
 }

.github/gh-actions-self-hosted-runners/arc/kubernetes.tf

@@ -27,6 +27,7 @@ resource "kubectl_manifest" "arc_autoscaler" {
   depends_on         = [helm_release.arc]
 }
 resource "kubectl_manifest" "arc_webhook_certificate" {
+  count = var.deploy_webhook != "false" ? 1 : 0
   yaml_body          = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain })
   override_namespace = "arc"
   depends_on         = [helm_release.arc]

.github/gh-actions-self-hosted-runners/arc/locals.tf

@@ -19,20 +19,18 @@
 
 
 locals {
-        subnetwork_cidr_range = "10.128.0.0/20"
         arc_values = {
-            "githubWebhookServer.enabled" = "true"
+            "githubWebhookServer.enabled" = "${var.deploy_webhook}"
             "authSecret.create" = "true"
             "authSecret.github_app_id" = data.google_secret_manager_secret_version.github_app_id.secret_data
             "authSecret.github_app_installation_id" = data.google_secret_manager_secret_version.github_app_install_id.secret_data
             "authSecret.github_app_private_key" = data.google_secret_manager_secret_version.github_private_key.secret_data
-            "githubWebhookServer.ingress.enabled" = "true"
+            "githubWebhookServer.ingress.enabled" = "${var.deploy_webhook}"
             "githubWebhookServer.ingress.hosts[0].host" = var.ingress_domain
             "githubWebhookServer.ingress.hosts[0].paths[0].path" = "/"
             "githubWebhookServer.ingress.hosts[0].paths[0].pathType" = "ImplementationSpecific"
             "githubWebhookServer.service.type" = "NodePort"
-            #"githubWebhookServer.ingress.tls[0].hosts[0]" = var.ingress_domain
-            "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = google_compute_global_address.actions-runner-ip.name
+            "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].name : "not-configured"
             "githubWebhookServer.ingress.annotations.networking\\.gke\\.io/managed-certificates" = "managed-cert"
             "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.class" = "gce"
         }

.github/gh-actions-self-hosted-runners/arc/network.tf

@@ -18,15 +18,21 @@
 #
 
 resource "google_compute_network" "actions-runner-network" {
+  count = var.existing_vpc_name == "" ? 1 : 0
   project = var.project_id
   name    = "${var.environment}-actions-runner-network"
   auto_create_subnetworks = false
 }
+data "google_compute_network" "actions-runner-network" {
+  name = var.existing_vpc_name == "" ? google_compute_network.actions-runner-network[0].name : var.existing_vpc_name
+  project = var.project_id
+}
+
 
 resource "google_compute_subnetwork" "actions-runner-subnetwork" {
-  ip_cidr_range            = local.subnetwork_cidr_range
+  ip_cidr_range            = var.subnetwork_cidr_range
   name                     = "${var.environment}-actions-runner-subnetwork"
-  network                  = google_compute_network.actions-runner-network.id
+  network                  = data.google_compute_network.actions-runner-network.id
   region                   = var.region
   project                  = var.project_id
 }

.github/gh-actions-self-hosted-runners/arc/outputs.tf

@@ -24,9 +24,8 @@ output "cluster_endpoint" {
     value = google_container_cluster.actions-runner-gke.endpoint
 }
 output "ingress_ip" {
-    value = google_compute_global_address.actions-runner-ip.address
+    value = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].address : "Not Configured"
 }
-
 output "get_kubeconfig_command" {
     value = "gcloud container clusters get-credentials ${google_container_cluster.actions-runner-gke.name} --region ${var.zone} --project ${var.project_id}"
 }

.github/gh-actions-self-hosted-runners/arc/provider.tf

@@ -19,7 +19,7 @@
 
 terraform {
    backend "gcs" {
-    prefix = "test-state"
+    prefix = "prod"
    }
 
   required_providers {
@@ -28,8 +28,8 @@ terraform {
       version = "~> 4.62.0"
     }
     kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = ">= 1.7.0"
+      source  = "alekc/kubectl"
+      version = ">= 2.0.2"
     }
   }
 }