From 459c87af706e64c5e4c71a004592fba1c3b1dea9 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 13 May 2024 15:28:48 -0400 Subject: [PATCH 1/3] Revert "[flink] Add documentation for Flink runner versions updates" This reverts commit 5ff55b1caf109ce1da44500598dfaf4e85ef1caa. --- .../setup-default-test-properties/test-properties.json | 2 +- sdks/go/examples/wasm/README.md | 2 +- sdks/python/apache_beam/options/pipeline_options.py | 2 +- sdks/typescript/src/apache_beam/runners/flink.ts | 2 +- settings.gradle.kts | 8 -------- .../www/site/content/en/documentation/runners/flink.md | 8 +------- 6 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup-default-test-properties/test-properties.json b/.github/actions/setup-default-test-properties/test-properties.json index fc43020666706..4a43463eb2f9d 100644 --- a/.github/actions/setup-default-test-properties/test-properties.json +++ b/.github/actions/setup-default-test-properties/test-properties.json @@ -14,7 +14,7 @@ }, "JavaTestProperties": { "SUPPORTED_VERSIONS": ["8", "11", "17", "21"], - "FLINK_VERSIONS": ["1.14", "1.15", "1.16", "1.17", "1.18"], + "FLINK_VERSIONS": ["1.14", "1.15", "1.16", "1.17"], "SPARK_VERSIONS": ["2", "3"] }, "GoTestProperties": { diff --git a/sdks/go/examples/wasm/README.md b/sdks/go/examples/wasm/README.md index e8802bdecb0ba..cb08165885024 100644 --- a/sdks/go/examples/wasm/README.md +++ b/sdks/go/examples/wasm/README.md @@ -68,7 +68,7 @@ cd $BEAM_HOME Expected output should include the following, from which you acquire the latest flink runner version. ```shell -'flink_versions: 1.14,1.15,1.16,1.17,1.18' +'flink_versions: 1.14,1.15,1.16,1.17' ``` #### 2. Set to the latest flink runner version i.e. 1.16 diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index 04ae94df78971..93cde0cf2d40e 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -1591,7 +1591,7 @@ def _add_argparse_args(cls, parser): class FlinkRunnerOptions(PipelineOptions): # These should stay in sync with gradle.properties. - PUBLISHED_FLINK_VERSIONS = ['1.14', '1.15', '1.16', '1.17', '1.18'] + PUBLISHED_FLINK_VERSIONS = ['1.14', '1.15', '1.16', '1.17'] @classmethod def _add_argparse_args(cls, parser): diff --git a/sdks/typescript/src/apache_beam/runners/flink.ts b/sdks/typescript/src/apache_beam/runners/flink.ts index bcccd0b468717..3b672b48105cd 100644 --- a/sdks/typescript/src/apache_beam/runners/flink.ts +++ b/sdks/typescript/src/apache_beam/runners/flink.ts @@ -28,7 +28,7 @@ import { JavaJarService } from "../utils/service"; const MAGIC_HOST_NAMES = ["[local]", "[auto]"]; // These should stay in sync with gradle.properties. -const PUBLISHED_FLINK_VERSIONS = ["1.14", "1.15", "1.16", "1.17", "1.18"]; +const PUBLISHED_FLINK_VERSIONS = ["1.14", "1.15", "1.16", "1.17"]; const defaultOptions = { flinkMaster: "[local]", diff --git a/settings.gradle.kts b/settings.gradle.kts index b5e977f31b2a5..973d9cf8c76e7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -121,14 +121,6 @@ include(":runners:core-java") include(":runners:direct-java") include(":runners:extensions-java:metrics") /* Begin Flink Runner related settings */ -/* When updating these versions, please make sure that the following files are updated as well: - * FLINK_VERSIONS in .github/actions/setup-default-test-properties/test-properties.json - * flink_versions in sdks/go/examples/wasm/README.md - * PUBLISHED_FLINK_VERSIONS in sdks/python/apache_beam/options/pipeline_options.py - * PUBLISHED_FLINK_VERSIONS in sdks/typescript/src/apache_beam/runners/flink.ts - * verify versions in website/www/site/content/en/documentation/runners/flink.md - * verify version in sdks/python/apache_beam/runners/interactive/interactive_beam.py - */ // Flink 1.14 include(":runners:flink:1.14") include(":runners:flink:1.14:job-server") diff --git a/website/www/site/content/en/documentation/runners/flink.md b/website/www/site/content/en/documentation/runners/flink.md index 53d5819ca9330..8ecb08e5fe92c 100644 --- a/website/www/site/content/en/documentation/runners/flink.md +++ b/website/www/site/content/en/documentation/runners/flink.md @@ -200,7 +200,6 @@ Starting with Beam 2.18.0, pre-built Flink Job Service Docker images are availab [Flink 1.15](https://hub.docker.com/r/apache/beam_flink1.15_job_server). [Flink 1.16](https://hub.docker.com/r/apache/beam_flink1.16_job_server). [Flink 1.17](https://hub.docker.com/r/apache/beam_flink1.17_job_server). -[Flink 1.18](https://hub.docker.com/r/apache/beam_flink1.18_job_server). {{< /paragraph >}} @@ -327,14 +326,9 @@ To find out which version of Flink is compatible with Beam please see the table Artifact Id Supported Beam Versions - - 1.18.x - beam-runners-flink-1.18 - ≥ 2.57.0 - 1.17.x - beam-runners-flink-1.17 + beam-runners-flink-1.16 ≥ 2.56.0 From 557b5bad1b0b198a13c27bc4e69b673d1b4e39ca Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 13 May 2024 15:30:01 -0400 Subject: [PATCH 2/3] Revert "[#30789] Add support for Flink 1.18 (#31062)" This reverts commit 45fe4f96f22fad059f61b2a157e1a9f89ad5e604. --- .../beam_PostCommit_Java_Tpcds_Flink.yml | 2 +- ..._PostCommit_Java_ValidatesRunner_Flink.yml | 2 +- CHANGES.md | 1 - gradle.properties | 2 +- runners/flink/1.18/build.gradle | 25 --------------- .../1.18/job-server-container/build.gradle | 26 ---------------- runners/flink/1.18/job-server/build.gradle | 31 ------------------- runners/flink/flink_runner.gradle | 2 +- sdks/go/examples/stringsplit/stringsplit.go | 2 +- settings.gradle.kts | 4 --- 10 files changed, 5 insertions(+), 92 deletions(-) delete mode 100644 runners/flink/1.18/build.gradle delete mode 100644 runners/flink/1.18/job-server-container/build.gradle delete mode 100644 runners/flink/1.18/job-server/build.gradle diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml index 2818f69507af1..b6d27fd3377f9 100644 --- a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml +++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml @@ -101,5 +101,5 @@ jobs: with: gradle-command: :sdks:java:testing:tpcds:run arguments: | - -Ptpcds.runner=:runners:flink:1.18 \ + -Ptpcds.runner=:runners:flink:1.17 \ "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \ diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml index ae028eb894ff7..1a3ddcf7e995c 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml @@ -78,7 +78,7 @@ jobs: - name: run validatesRunner script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :runners:flink:1.18:validatesRunner + gradle-command: :runners:flink:1.17:validatesRunner - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/CHANGES.md b/CHANGES.md index 610df5d15e66e..bce9636237e36 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -57,7 +57,6 @@ ## Highlights -* Added FlinkRunner for Flink 1.18. ([#30789](https://github.com/apache/beam/issues/30789)) * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)). * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)). diff --git a/gradle.properties b/gradle.properties index 8c41a9f86c8f1..0d33c30d3f780 100644 --- a/gradle.properties +++ b/gradle.properties @@ -39,6 +39,6 @@ docker_image_default_repo_root=apache docker_image_default_repo_prefix=beam_ # supported flink versions -flink_versions=1.14,1.15,1.16,1.17,1.18 +flink_versions=1.14,1.15,1.16,1.17 # supported python versions python_versions=3.8,3.9,3.10,3.11 diff --git a/runners/flink/1.18/build.gradle b/runners/flink/1.18/build.gradle deleted file mode 100644 index ab6e6b63b7734..0000000000000 --- a/runners/flink/1.18/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -project.ext { - flink_major = '1.18' - flink_version = '1.18.0' -} - -// Load the main build script which contains all build logic. -apply from: "../flink_runner.gradle" diff --git a/runners/flink/1.18/job-server-container/build.gradle b/runners/flink/1.18/job-server-container/build.gradle deleted file mode 100644 index afdb68a0fc91c..0000000000000 --- a/runners/flink/1.18/job-server-container/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -def basePath = '../../job-server-container' - -project.ext { - resource_path = basePath -} - -// Load the main build script which contains all build logic. -apply from: "$basePath/flink_job_server_container.gradle" diff --git a/runners/flink/1.18/job-server/build.gradle b/runners/flink/1.18/job-server/build.gradle deleted file mode 100644 index e70fdcc0c5819..0000000000000 --- a/runners/flink/1.18/job-server/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -def basePath = '../../job-server' - -project.ext { - // Look for the source code in the parent module - main_source_dirs = ["$basePath/src/main/java"] - test_source_dirs = ["$basePath/src/test/java"] - main_resources_dirs = ["$basePath/src/main/resources"] - test_resources_dirs = ["$basePath/src/test/resources"] - archives_base_name = 'beam-runners-flink-1.18-job-server' -} - -// Load the main build script which contains all build logic. -apply from: "$basePath/flink_job_server.gradle" diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index af60caf5d145d..3965ab3b1ba03 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -376,7 +376,7 @@ tasks.register("validatesRunnerSickbay", Test) { } } -// Generates :runners:flink:1.18:runQuickstartJavaFlinkLocal +// Generates :runners:flink:1.17:runQuickstartJavaFlinkLocal createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal') tasks.register("examplesIntegrationTest", Test) { diff --git a/sdks/go/examples/stringsplit/stringsplit.go b/sdks/go/examples/stringsplit/stringsplit.go index 266cdd99fb373..28d79815607f2 100644 --- a/sdks/go/examples/stringsplit/stringsplit.go +++ b/sdks/go/examples/stringsplit/stringsplit.go @@ -21,7 +21,7 @@ // 1. From a command line, navigate to the top-level beam/ directory and run // the Flink job server: // -// ./gradlew :runners:flink:1.18:job-server:runShadow -Djob-host=localhost -Dflink-master=local +// ./gradlew :runners:flink:1.17:job-server:runShadow -Djob-host=localhost -Dflink-master=local // // 2. The job server is ready to receive jobs once it outputs a log like the // following: `JobService started on localhost:8099`. Take note of the endpoint diff --git a/settings.gradle.kts b/settings.gradle.kts index 973d9cf8c76e7..a040e7e578d89 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -137,10 +137,6 @@ include(":runners:flink:1.16:job-server-container") include(":runners:flink:1.17") include(":runners:flink:1.17:job-server") include(":runners:flink:1.17:job-server-container") -// Flink 1.18 -include(":runners:flink:1.18") -include(":runners:flink:1.18:job-server") -include(":runners:flink:1.18:job-server-container") /* End Flink Runner related settings */ include(":runners:twister2") include(":runners:google-cloud-dataflow-java") From 0542c83fc1a239957e8170b93ed5500cc68081ad Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 13 May 2024 14:32:09 -0400 Subject: [PATCH 3/3] Un-revert doc fixes --- settings.gradle.kts | 12 ++++++++++++ .../site/content/en/documentation/runners/flink.md | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index a040e7e578d89..b5e977f31b2a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -121,6 +121,14 @@ include(":runners:core-java") include(":runners:direct-java") include(":runners:extensions-java:metrics") /* Begin Flink Runner related settings */ +/* When updating these versions, please make sure that the following files are updated as well: + * FLINK_VERSIONS in .github/actions/setup-default-test-properties/test-properties.json + * flink_versions in sdks/go/examples/wasm/README.md + * PUBLISHED_FLINK_VERSIONS in sdks/python/apache_beam/options/pipeline_options.py + * PUBLISHED_FLINK_VERSIONS in sdks/typescript/src/apache_beam/runners/flink.ts + * verify versions in website/www/site/content/en/documentation/runners/flink.md + * verify version in sdks/python/apache_beam/runners/interactive/interactive_beam.py + */ // Flink 1.14 include(":runners:flink:1.14") include(":runners:flink:1.14:job-server") @@ -137,6 +145,10 @@ include(":runners:flink:1.16:job-server-container") include(":runners:flink:1.17") include(":runners:flink:1.17:job-server") include(":runners:flink:1.17:job-server-container") +// Flink 1.18 +include(":runners:flink:1.18") +include(":runners:flink:1.18:job-server") +include(":runners:flink:1.18:job-server-container") /* End Flink Runner related settings */ include(":runners:twister2") include(":runners:google-cloud-dataflow-java") diff --git a/website/www/site/content/en/documentation/runners/flink.md b/website/www/site/content/en/documentation/runners/flink.md index 8ecb08e5fe92c..94e6036af87b8 100644 --- a/website/www/site/content/en/documentation/runners/flink.md +++ b/website/www/site/content/en/documentation/runners/flink.md @@ -93,7 +93,7 @@ from the [compatibility table](#flink-version-compatibility) below. For example: {{< highlight java >}} org.apache.beam - beam-runners-flink-1.16 + beam-runners-flink-1.17 {{< param release_latest >}} {{< /highlight >}} @@ -328,7 +328,7 @@ To find out which version of Flink is compatible with Beam please see the table 1.17.x - beam-runners-flink-1.16 + beam-runners-flink-1.17 ≥ 2.56.0