Initial DaskRunner for Beam.

.asf.yaml

@@ -37,6 +37,8 @@ github:
   # Give some users issue triage permissions
   collaborators:
     - pcoet
+    - VladMatyunin
+    - olehborysevych
 
   enabled_merge_buttons:
     squash: true

.github/REVIEWERS.yml

@@ -23,9 +23,9 @@ labels:
       - damccorm
       - lostluck
       - jrmccluskey
-      - youngoli
       - riteshghorse
-    exclusionList: []
+    exclusionList:
+      - youngoli
   - name: Python
     reviewers:
       - AnandInguva
@@ -36,4 +36,25 @@ labels:
       - pabloem
       - y1chi
     exclusionList: []
+  - name: Java
+    reviewers:
+      - lukecwik
+      - kennknowles
+      - robertwb
+      - kileys
+      - apilloud
+    exclusionList: []
+  - name: IO
+    reviewers:
+      - chamikaramj
+      - johnjcasey
+      - pabloem
+      - Abacn
+      - ahmedabu98
+    exclusionList: []
+  - name: Build
+    reviewers:
+      - damccorm
+      - Abacn
+    exclusionList: []
 fallbackReviewers: []

.github/autolabeler.yml

@@ -42,6 +42,7 @@ extensions: ["sdks/java/extensions/**/*", "runners/extensions-java/**/*"]
 "sketching": ["sdks/java/extensions/sketching/**/*"]
 "sorter": ["sdks/java/extensions/sorter/**/*"]
 "sql": ["sdks/java/extensions/sql/**/*"]
+"timeseries": ["sdks/java/extensions/timeseries/*"]
 "zetasketch": ["sdks/java/extensions/zetasketch/**/*"]
 
 # IO

.github/workflows/build_playground_backend.yml

@@ -51,6 +51,7 @@ jobs:
         uses: google-github-actions/setup-gcloud@v0
         with:
             install_components: 'beta,cloud-datastore-emulator'
+            version: '389.0.0'
 
       - name: Run PreCommit
         run: ./gradlew playground:backend:precommit

.github/workflows/playground_deploy_examples.yml

@@ -16,11 +16,10 @@
 name: Collect And Deploy Playground Examples
 
 on:
-  push:
-    tags: 'v*'
-    branches: ['master', 'release-*']
   pull_request:
-  workflow_dispatch:
+    types:
+      - closed
+    branches: ['master']
 env:
   BEAM_ROOT_DIR: ../../
   BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
@@ -48,7 +47,8 @@ jobs:
         name: get Difference
         id: check_file_changed
         run: |
-          $diff = git diff --name-only master
+          $diff = git diff --name-only ${{ github.event.before }} ${{ github.event.after }}
+          echo "$diff"
           Write-Host "::set-output name=example_diff::$diff"
       - name: has Examples
         run: |
@@ -95,7 +95,7 @@ jobs:
         run: |
              echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json
              which gcloud
-             gcloud auth activate-service-account --project=apache-beam-testing --key-file=/tmp/gcp_access.json
+             gcloud auth activate-service-account --project=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} --key-file=/tmp/gcp_access.json
       - name: Set Docker Tag
         run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV
       - name: Set Docker Tag If Github Tag was trigger

.github/workflows/pr-bot-new-prs.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - run: npm install
+      - run: npm ci
         working-directory: 'scripts/ci/pr-bot'
 
       # Runs a set of commands using the runners shell

.github/workflows/pr-bot-pr-updates.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           ref: 'master'
-      - run: npm install
+      - run: npm ci
         working-directory: 'scripts/ci/pr-bot'
 
       # Runs a set of commands using the runners shell

.github/workflows/pr-bot-prs-needing-attention.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - run: npm install
+      - run: npm ci
         working-directory: 'scripts/ci/pr-bot'
 
       # Runs a set of commands using the runners shell

.github/workflows/pr-bot-update-reviewers.yml

@@ -29,7 +29,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - run: npm install
+      - run: npm ci
         working-directory: 'scripts/ci/pr-bot'
 
       # Runs a set of commands using the runners shell

.github/workflows/reportGenerator.yml

@@ -28,7 +28,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - run: |
-        npm install
+        npm ci
         node generateReport.js
       working-directory: 'scripts/ci/issue-report'
       env:

.github/workflows/self-assign.yml

@@ -26,7 +26,8 @@ jobs:
     - uses: actions/github-script@v6
       with:
         script: |
-          const body = context.payload.comment.body.split(' ');
+          const body = context.payload.comment.body.replace( /\r\n/g, " " ).replace( /\n/g, " " ).split(' ');
+          console.log(body);
           for (let i = 0; i < body.length; i++) {
             const bodyString = body[i].toLowerCase();
             if (bodyString == '.take-issue') {

.github/workflows/typescript_tests.yml

@@ -51,7 +51,7 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: '16'
-      - run: npm install
+      - run: npm ci
         working-directory: ./sdks/typescript
       - run: npm run build
         working-directory: ./sdks/typescript

.test-infra/jenkins/committers_list_generator/main.py

@@ -63,7 +63,6 @@ def get_committers_github_usernames(ldap_uris, cert_path):
     connection = None
     try:
         ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, cert_path)
-        ldap.set_option(ldap.OPT_X_TLS, ldap.OPT_X_TLS_DEMAND)
         ldap.set_option(ldap.OPT_X_TLS_DEMAND, True)
         ldap.set_option(ldap.OPT_REFERRALS, 0)
         connection = ldap.initialize(ldap_uris)

.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy

@@ -42,6 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja
           // overloading the machines.
           commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
           switches '-PdisableSpotlessCheck=true'
+          switches '-PdisableCheckStyle=true'
           switches '-PcompileAndRunTestsWithJava11'
           switches '-PskipCheckerFramework'
           switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}"

.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy

@@ -42,6 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja
           // overloading the machines.
           commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
           switches '-PdisableSpotlessCheck=true'
+          switches '-PdisableCheckStyle=true'
           switches '-PcompileAndRunTestsWithJava17'
           switches '-PskipCheckerFramework'
           switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}"

.test-infra/jenkins/job_PreCommit_Java.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java',
     gradleTask: ':javaPreCommit',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     timeoutMins: 180,
     triggerPathPatterns: [
@@ -45,9 +46,6 @@ builder.build {
       tools {
         errorProne()
         java()
-        checkStyle {
-          pattern('**/build/reports/checkstyle/*.xml')
-        }
         spotBugs {
           pattern('**/build/reports/spotbugs/*.xml')
         }

.test-infra/jenkins/job_PreCommit_Java_AWS2_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Amazon-Web-Services2_IO_Direct',
     gradleTask: 'sdks:java:io:amazon-web-services2:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/amazon-web-services2/.*$',

.test-infra/jenkins/job_PreCommit_Java_AWS_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Amazon-Web-Services_IO_Direct',
     gradleTask: ':sdks:java:io:amazon-web-services:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/amazon-web-services/.*$',

.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Debezium_IO_Direct',
     gradleTask: ':sdks:java:io:debezium:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/debezium/.*$',

.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Examples_Dataflow',
     gradleTask: ':javaExamplesDataflowPreCommit',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^model/.*$',

.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy

@@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     gradleTask: ':clean',
     gradleSwitches: [
       '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true',
       '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
@@ -47,6 +48,7 @@ builder.build {
       rootBuildScriptDir(properties.checkoutDir)
       tasks 'javaExamplesDataflowPreCommit'
       switches '-PdisableSpotlessCheck=true'
+      switches '-PdisableCheckStyle=true'
       switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
       switches '-PcompileAndRunTestsWithJava11'
       switches "-Pjava11Home=${properties.JAVA_11_HOME}"

.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_GCP_IO_Direct',
     gradleTask: ':sdks:java:io:google-cloud-platform:postCommit',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     timeoutMins: 120,
     triggerPathPatterns: [

.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_JDBC_IO_Direct',
     gradleTask: ':sdks:java:io:jdbc:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/jdbc/.*$',

.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Kafka_IO_Direct',
     gradleTask: ':sdks:java:io:kafka:kafkaVersionsCompatibilityTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/kafka/.*$',

.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Kinesis_IO_Direct',
     gradleTask: ':sdks:java:io:kinesis:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/kinesis/.*$',

.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java_Neo4j_IO_Direct',
     gradleTask: ':sdks:java:io:neo4j:integrationTest',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
       '^sdks/java/io/neo4j/.*$',

.test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+    scope: this,
+    nameBase: 'Java_Spark3_Versions',
+    gradleTask: ':runners:spark:3:sparkVersionsTest',
+    gradleSwitches: [
+      '-PdisableSpotlessCheck=true'
+    ], // spotless checked in separate pre-commit
+    triggerPathPatterns: [
+      '^runners/spark/.*$',
+    ],
+    timeoutMins: 120,
+    )
+builder.build {
+  publishers {
+    archiveJunit('**/build/test-results/**/*.xml')
+  }
+}

.test-infra/jenkins/job_PreCommit_SQL.groovy

@@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'SQL',
     gradleTask: ':sqlPreCommit',
     gradleSwitches: [
-      '-PdisableSpotlessCheck=true'
+      '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true'
     ], // spotless checked in job_PreCommit_Spotless
     triggerPathPatterns: [
       '^sdks/java/extensions/sql.*$',
@@ -35,10 +36,6 @@ builder.build {
     recordIssues {
       tools {
         errorProne()
-        java()
-        checkStyle {
-          pattern('**/build/reports/checkstyle/*.xml')
-        }
         spotBugs {
           pattern('**/build/reports/spotbugs/*.xml')
         }

.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy

@@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     gradleTask: ':sqlPreCommit',
     gradleSwitches: [
       '-PdisableSpotlessCheck=true',
+      '-PdisableCheckStyle=true',
       '-PcompileAndRunTestsWithJava11',
       '-PskipCheckerFramework',
       // Gradle itself is running under JDK8 so plugin configures wrong for JDK11
@@ -41,9 +42,6 @@ builder.build {
       tools {
         errorProne()
         java()
-        checkStyle {
-          pattern('**/build/reports/checkstyle/*.xml')
-        }
         spotBugs {
           pattern('**/build/reports/spotbugs/*.xml')
         }