From fa6e3ee3b83aa5ffc070557780ec111386b8c951 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 3 Apr 2024 15:44:52 +0800 Subject: [PATCH] [ZEPPELIN-5969] Remove Hadoop2 and move to Hadoop3 shaded client (#4691) * Drop hadoop2 in github actions * Update docs * Drop hadoop2 support * Remove hadoop2 integration tests * findbugs use the same version in all modules * Use hadoop3.3 for tests * Move to scala 2.12 * Try to fix flink * Usage of metals * Remove duplicate version and groupid * Fix Flink with Hadoop3 * fix log * R * fix * fix * fix * fix * hadoop-3.3 * fix * fix * Address comments * address comments --------- Co-authored-by: Philipp Dallig --- .github/workflows/core.yml | 20 +- .github/workflows/frontend.yml | 8 +- .github/workflows/quick.yml | 2 +- .gitignore | 4 + alluxio/pom.xml | 16 +- docs/interpreter/flink.md | 2 +- docs/setup/basics/how_to_build.md | 1 - .../deployment/flink_and_spark_cluster.md | 16 +- flink-cmd/pom.xml | 9 +- flink/flink-scala-2.12/pom.xml | 695 ++++++++++--- .../main/resources/interpreter-setting.json | 2 +- .../flink/FlinkScalaInterpreter.scala | 2 +- .../flink/FlinkSqlInterpreterTest.java | 2 +- flink/flink1.15-shims/pom.xml | 2 - flink/flink1.16-shims/pom.xml | 2 - hbase/pom.xml | 41 - jdbc/pom.xml | 208 +--- livy/pom.xml | 120 +-- pom.xml | 978 +----------------- rlang/pom.xml | 21 +- spark/interpreter/pom.xml | 30 - spark/spark-scala-parent/pom.xml | 15 - spark/spark-shims/pom.xml | 3 +- submarine/pom.xml | 53 +- zeppelin-interpreter-integration/pom.xml | 22 +- .../integration/FlinkIntegrationTest.java | 7 +- .../integration/SparkIntegrationTest32.java | 11 - .../integration/SparkIntegrationTest33.java | 10 - .../integration/SparkIntegrationTest34.java | 12 +- .../integration/ZeppelinFlinkClusterTest.java | 2 - .../ZeppelinSparkClusterTest32.java | 10 - .../ZeppelinSparkClusterTest33.java | 10 - zeppelin-interpreter/pom.xml | 61 +- zeppelin-plugins/launcher/yarn/pom.xml | 32 +- .../notebookrepo/filesystem/pom.xml | 110 +- zeppelin-plugins/notebookrepo/s3/pom.xml | 24 - zeppelin-server/pom.xml | 20 +- zeppelin-zengine/pom.xml | 67 +- .../integration/DownloadUtils.java | 7 +- 39 files changed, 693 insertions(+), 1964 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index b197fd16ebd..e441c823844 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -40,7 +40,7 @@ jobs: strategy: fail-fast: false matrix: - hadoop: [hadoop2, hadoop3] + hadoop: [hadoop3] java: [ 8, 11 ] steps: - name: Checkout @@ -183,7 +183,7 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - ./mvnw install -DskipTests -pl python,rlang,zeppelin-jupyter-interpreter -am -Phadoop2 ${MAVEN_ARGS} + ./mvnw install -DskipTests -pl python,rlang,zeppelin-jupyter-interpreter -am -Phadoop3 ${MAVEN_ARGS} - name: run tests with ${{ matrix.python }} run: | ./mvnw test -pl python,rlang,zeppelin-jupyter-interpreter -DfailIfNoTests=false ${MAVEN_ARGS} @@ -221,7 +221,7 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: | - ./mvnw install -DskipTests -Phadoop2 -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pflink-117 ${MAVEN_ARGS} + ./mvnw install -DskipTests -Phadoop3 -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.12,jdbc,shell -am -Pflink-117 ${MAVEN_ARGS} ./mvnw package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - name: Setup conda environment with python 3.9 and R uses: conda-incubator/setup-miniconda@v2 @@ -238,7 +238,7 @@ jobs: run: | R -e "IRkernel::installspec()" - name: run tests - run: ./mvnw test -pl zeppelin-interpreter-integration -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest + run: ./mvnw test -pl zeppelin-interpreter-integration -Phadoop3 -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest - name: Print zeppelin logs if: always() run: if [ -d "logs" ]; then cat logs/*; fi @@ -278,7 +278,7 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment for flink run: | - ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration ${MAVEN_ARGS} + ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop3 -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 @@ -292,7 +292,7 @@ jobs: auto-activate-base: false use-mamba: true - name: run tests for flink - run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} + run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Phadoop3 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} - name: Print zeppelin logs if: always() run: if [ -d "logs" ]; then cat logs/*; fi @@ -328,7 +328,7 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: | - ./mvnw install -DskipTests -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown -am -Phadoop2 -Pintegration ${MAVEN_ARGS} + ./mvnw install -DskipTests -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.12,spark/scala-2.13,markdown -am -Phadoop3 -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - name: Setup conda environment with python 3.9 and R uses: conda-incubator/setup-miniconda@v2 @@ -382,7 +382,7 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: install environment - run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am -Phadoop2 ${MAVEN_ARGS} + run: ./mvnw install -DskipTests -pl spark-submit,spark/scala-2.12,spark/scala-2.13 -am -Phadoop3 ${MAVEN_ARGS} - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 with: @@ -400,11 +400,11 @@ jobs: - name: run spark-3.2 tests with scala-2.12 and python-${{ matrix.python }} run: | rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} + ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.12 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - name: run spark-3.2 tests with scala-2.13 and python-${{ matrix.python }} run: | rm -rf spark/interpreter/metastore_db - ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.13 -Phadoop2 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} + ./mvnw verify -pl spark-submit,spark/interpreter -am -Dtest=org/apache/zeppelin/spark/* -Pspark-3.2 -Pspark-scala-2.13 -Phadoop3 -Pintegration -DfailIfNoTests=false ${MAVEN_ARGS} - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} run: | rm -rf spark/interpreter/metastore_db diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index d7fcbc2fa76..75175e45592 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -53,9 +53,9 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: Install application - run: ./mvnw clean install -DskipTests -am -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist ${MAVEN_ARGS} + run: ./mvnw clean install -DskipTests -am -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist ${MAVEN_ARGS} - name: Run headless test - run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist -Pweb-e2e ${MAVEN_ARGS} + run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -pl zeppelin-web -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist -Pweb-e2e ${MAVEN_ARGS} - name: Print zeppelin logs if: always() run: if [ -d "logs" ]; then cat logs/*; fi @@ -128,10 +128,10 @@ jobs: R -e "IRkernel::installspec()" - name: Install Environment run: | - ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist ${MAVEN_ARGS} + ./mvnw clean install -DskipTests -am -pl zeppelin-integration -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist ${MAVEN_ARGS} - name: run tests run: | - source ./testing/downloadSpark.sh "3.4.1" "3" && echo "SPARK_HOME: ${SPARK_HOME}" && xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" ./mvnw verify -DfailIfNoTests=false -pl zeppelin-integration -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop2 -Pweb-dist -Pusing-source-tree ${MAVEN_ARGS} + source ./testing/downloadSpark.sh "3.4.1" "3" && echo "SPARK_HOME: ${SPARK_HOME}" && xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" ./mvnw verify -DfailIfNoTests=false -pl zeppelin-integration -Pintegration -Pspark-scala-2.12 -Pspark-3.4 -Phadoop3 -Pweb-dist -Pusing-source-tree ${MAVEN_ARGS} - name: Print zeppelin logs if: always() run: if [ -d "logs" ]; then cat logs/*; fi diff --git a/.github/workflows/quick.yml b/.github/workflows/quick.yml index 5bc8ac17356..1b01389e558 100644 --- a/.github/workflows/quick.yml +++ b/.github/workflows/quick.yml @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - hadoop: [hadoop2, hadoop3] + hadoop: [hadoop3] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index 29bb190ebca..6784673088e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ /interpreter/* !/interpreter/lib +# metals +.bloop +.metals + # interpreter temp files derby.log spark/metastore_db diff --git a/alluxio/pom.xml b/alluxio/pom.xml index 55f1a34a6aa..87d374db3e3 100644 --- a/alluxio/pom.xml +++ b/alluxio/pom.xml @@ -68,15 +68,27 @@ alluxio-minicluster ${alluxio.version} test + + + org.apache.hadoop + hadoop-client + + org.apache.hadoop - hadoop-common - 3.2.4 + hadoop-client-api + ${hadoop.version} test + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + test + diff --git a/docs/interpreter/flink.md b/docs/interpreter/flink.md index cc40d03a7aa..df272cbdb9b 100644 --- a/docs/interpreter/flink.md +++ b/docs/interpreter/flink.md @@ -301,7 +301,7 @@ You can also add and set other Flink properties which are not listed in the tabl zeppelin.flink.hive.version - 2.3.4 + 2.3.7 Hive version that you would like to connect diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md index b131ec57d10..8c8cd947f84 100644 --- a/docs/setup/basics/how_to_build.md +++ b/docs/setup/basics/how_to_build.md @@ -123,7 +123,6 @@ Set hadoop major version (default hadoop3). Available profiles are ``` --Phadoop2 -Phadoop3 ``` diff --git a/docs/setup/deployment/flink_and_spark_cluster.md b/docs/setup/deployment/flink_and_spark_cluster.md index d395ccab67f..76f9063cf13 100644 --- a/docs/setup/deployment/flink_and_spark_cluster.md +++ b/docs/setup/deployment/flink_and_spark_cluster.md @@ -225,16 +225,16 @@ Building from source is recommended where possible, for simplicity in this tuto To download the Flink Binary use `wget` ```bash -wget "http://mirror.cogentco.com/pub/apache/flink/flink-1.1.3/flink-1.1.3-bin-hadoop24-scala_2.10.tgz" -tar -xzvf flink-1.1.3-bin-hadoop24-scala_2.10.tgz +wget "http://mirror.cogentco.com/pub/apache/flink/flink-1.16.2/flink-1.16.2-bin-scala_2.12.tgz" +tar -xzvf flink-1.16.2-bin-scala_2.12.tgz ``` -This will download Flink 1.1.3, compatible with Hadoop 2.4. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `24` to your appropriate version. +This will download Flink 1.16.2. Start the Flink Cluster. ```bash -flink-1.1.3/bin/start-cluster.sh +flink-1.16.2/bin/start-cluster.sh ``` ###### Building From source @@ -295,12 +295,12 @@ Using binaries is also To download the Spark Binary use `wget` ```bash -wget "http://d3kbcqa49mib13.cloudfront.net/spark-1.6.3-bin-hadoop2.6.tgz" -tar -xzvf spark-1.6.3-bin-hadoop2.6.tgz -mv spark-1.6.3-bin-hadoop2.6 spark +wget "https://dlcdn.apache.org/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz" +tar -xzvf spark-3.4.1-bin-hadoop3.tgz +mv spark-3.4.1-bin-hadoop3 spark ``` -This will download Spark 1.6.3, compatible with Hadoop 2.6. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `2.6` to your appropriate version. +This will download Spark 3.4.1, compatible with Hadoop 3. You do not have to install Hadoop for this binary to work, but if you are using Hadoop, please change `3` to your appropriate version. ###### Building From source diff --git a/flink-cmd/pom.xml b/flink-cmd/pom.xml index 0e82f083578..4f4456f8dc7 100644 --- a/flink-cmd/pom.xml +++ b/flink-cmd/pom.xml @@ -44,14 +44,7 @@ org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - - org.apache.hadoop - hadoop-yarn-client + hadoop-client-runtime ${hadoop.version} provided diff --git a/flink/flink-scala-2.12/pom.xml b/flink/flink-scala-2.12/pom.xml index 43b822f57b9..d711c9cc443 100644 --- a/flink/flink-scala-2.12/pom.xml +++ b/flink/flink-scala-2.12/pom.xml @@ -38,14 +38,14 @@ 2.12.7 2.12 ${flink.scala.version} - ${hadoop2.7.version} - 2.3.4 - 4.0.0 + 2.3.7 + 14.0.1 + 10.14.2.0 + 5.3.0 1.15.0 https://archive.apache.org/dist/flink/flink-${flink.version}/flink-${flink.version}-bin-scala_${flink.scala.binary.version}.tgz - @@ -160,10 +160,6 @@ org.apache.hadoop * - - org.eclipse.jetty - * - @@ -289,22 +285,12 @@ ${flink.version} tests test - - - org.apache.hive - hive-exec - - - org.apache.hive - hive-metastore - - - org.apache.hadoop - hadoop-common - ${flink.hadoop.version} + com.google.guava + guava + ${hive.guava.version} provided @@ -314,40 +300,6 @@ - - org.apache.hadoop - hadoop-hdfs - ${flink.hadoop.version} - provided - - - - org.apache.hadoop - hadoop-yarn-common - ${flink.hadoop.version} - provided - - - - org.apache.hadoop - hadoop-yarn-client - ${flink.hadoop.version} - provided - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${flink.hadoop.version} - provided - - - - org.mockito - mockito-core - test - - org.apache.hive hive-metastore @@ -355,32 +307,156 @@ provided - hadoop-auth - org.apache.hadoop + org.apache.hive + hive-shims + + + javolution + javolution com.google.guava guava - io.netty - netty + com.google.protobuf + protobuf-java - io.netty - netty-all + org.apache.hbase + hbase-client - com.google.protobuf - protobuf-java + commons-lang + commons-lang + + + co.cask.tephra + tephra-api + + + co.cask.tephra + tephra-core + + + co.cask.tephra + tephra-hbase-compat-1.0 + + + commons-cli + commons-cli + + + org.apache.thrift + libfb303 + + + javax.transaction + transaction-api + + + org.apache.orc + orc-core + + + joda-time + joda-time + + + org.apache.logging.log4j + log4j-1.2-api org.apache.logging.log4j log4j-slf4j-impl - jdk.tools - jdk.tools + org.apache.ant + ant + + + com.tdunning + json + + + jline + jline + + + org.eclipse.jetty.aggregate + jetty-all + + + org.eclipse.jetty.orbit + javax.servlet + + + org.apache.logging.log4j + log4j-web + + + io.dropwizard.metrics + metrics-core + + + io.dropwizard.metrics + metrics-jvm + + + io.dropwizard.metrics + metrics-json + + + com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter + + + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + + + commons-codec + commons-codec + + + org.apache.avro + avro + + + net.sf.opencsv + opencsv + + + org.apache.parquet + parquet-hadoop-bundle + + + org.slf4j + slf4j-log4j12 + + + org.apache.derby + derby + + + org.pentaho + pentaho-aggdesigner-algorithm @@ -392,16 +468,16 @@ provided - org.apache.calcite - calcite-core + org.apache.hive + hive-vector-code-gen - org.apache.calcite - calcite-druid + org.apache.hive + hive-llap-tez - org.apache.calcite.avatica - avatica + org.apache.hive + hive-shims commons-codec @@ -412,20 +488,28 @@ commons-httpclient - commons-io - commons-io + org.apache.logging.log4j + log4j-slf4j-impl + + + org.antlr + antlr-runtime - org.apache.logging.log4j - log4j-1.2-api + org.antlr + ST4 - org.apache.logging.log4j - log4j-slf4j-impl + org.apache.ant + ant - org.slf4j - slf4j-api + org.apache.commons + commons-compress + + + org.apache.ivy + ivy org.apache.zookeeper @@ -433,35 +517,63 @@ org.apache.curator - curator-framework + apache-curator org.apache.curator - apache-curator + curator-framework + + + org.codehaus.groovy + groovy-all + + + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-druid + + + org.apache.calcite.avatica + avatica + + + org.apache.calcite + calcite-avatica com.google.code.gson gson - jline - jline + stax + stax-api com.google.guava guava - io.netty - netty + log4j + log4j - io.netty - netty-all + log4j + apache-log4j-extras - com.google.protobuf - protobuf-java + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j org.pentaho @@ -471,36 +583,222 @@ - org.apache.hive.hcatalog - hive-webhcat-java-client - ${hive.version} + com.klarna + hiverunner + ${hiverunner.version} test - org.apache.calcite - * + org.apache.hive + hive-serde - com.google.guava - guava + org.apache.hive + hive-jdbc - io.netty - netty + org.apache.hive + hive-service + + + org.apache.hive + hive-contrib + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-hcatalog-core + + + org.apache.hive.hcatalog + hive-webhcat-java-client + + + org.apache.tez + tez-common + + + + jdk.tools + jdk.tools + + + hadoop-common + org.apache.hadoop + + + hadoop-auth + org.apache.hadoop + + + hadoop-annotations + org.apache.hadoop + + + hadoop-hdfs + org.apache.hadoop + + + hadoop-mapreduce-client-common + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-yarn-client + org.apache.hadoop + + + hadoop-yarn-common + org.apache.hadoop + + + hadoop-yarn-server-common + org.apache.hadoop + + + hadoop-yarn-server-web-proxy + org.apache.hadoop + + + hadoop-shim + org.apache.tez - javax.jms jms + javax.jms + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j org.apache.hive - hive-contrib + hive-service ${hive.version} test + + org.jamon + jamon-runtime + + + org.apache.hive + hive-exec + + + org.apache.hive + hive-metastore + + + com.google.guava + guava + + + + jdk.tools + jdk.tools + + + hadoop-common + org.apache.hadoop + + + hadoop-auth + org.apache.hadoop + + + hadoop-client + org.apache.hadoop + + + hadoop-annotations + org.apache.hadoop + + + hadoop-hdfs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-yarn-common + org.apache.hadoop + + + hadoop-yarn-registry + org.apache.hadoop + + + hadoop-yarn-server-applicationhistoryservice + org.apache.hadoop + + + hadoop-yarn-server-common + org.apache.hadoop + + + hadoop-yarn-server-resourcemanager + org.apache.hadoop + + + hbase-hadoop-compat + org.apache.hbase + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-server + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + org.pentaho pentaho-aggdesigner-algorithm @@ -513,27 +811,62 @@ hive-hcatalog-core ${hive.version} test - test-jar - jline - jline + org.jamon + jamon-runtime + + + org.apache.hive + hive-exec com.google.guava guava - io.netty - netty + hadoop-common + org.apache.hadoop - io.netty - netty-all + hadoop-archives + org.apache.hadoop - org.apache.logging.log4j - log4j-slf4j-impl + hadoop-annotations + org.apache.hadoop + + + hadoop-hdfs + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + log4j + log4j + + + log4j + apache-log4j-extras + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j org.pentaho @@ -543,21 +876,91 @@ - net.jodah - concurrentunit - 0.4.6 + org.apache.hive.hcatalog + hive-webhcat-java-client + ${hive.version} test + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.jamon + jamon-runtime + + + jdk.tools + jdk.tools + + + jms + javax.jms + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + org.pentaho + pentaho-aggdesigner-algorithm + + + - com.klarna - hiverunner - ${hiverunner.version} + org.apache.derby + derby + ${derby.version} + test + + + + org.apache.hive + hive-jdbc + ${hive.version} test - com.google.guava - guava + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + org.jamon + jamon-runtime + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j org.pentaho @@ -567,9 +970,27 @@ - org.scalatest - scalatest_${flink.scala.binary.version} - 3.0.8 + org.apache.hadoop + hadoop-client-runtime + provided + + + + org.mockito + mockito-core + test + + + + org.assertj + assertj-core + test + + + + net.jodah + concurrentunit + 0.4.6 test @@ -578,6 +999,14 @@ slf4j-api + + + com.lmax + disruptor + 3.4.4 + test + + org.junit.jupiter junit-jupiter-engine @@ -904,31 +1333,5 @@ - - - hive2 - - true - - - 2.3.4 - 4.0.0 - - - - hive1 - - 1.2.1 - 3.2.1 - - - - org.apache.hadoop - hadoop-common - 2.7.5 - provided - - - diff --git a/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json b/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json index 9da270aace4..f5968526c7d 100644 --- a/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json +++ b/flink/flink-scala-2.12/src/main/resources/interpreter-setting.json @@ -155,7 +155,7 @@ "zeppelin.flink.hive.version": { "envName": null, "propertyName": null, - "defaultValue": "2.3.4", + "defaultValue": "2.3.7", "description": "Hive version that you would like to connect", "type": "string" }, diff --git a/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala b/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala index c6ce85c9857..16d38f0b43f 100644 --- a/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala +++ b/flink/flink-scala-2.12/src/main/scala/org/apache/zeppelin/flink/FlinkScalaInterpreter.scala @@ -491,7 +491,7 @@ abstract class FlinkScalaInterpreter(val properties: Properties, throw new InterpreterException("HIVE_CONF_DIR is not specified"); } val database = properties.getProperty("zeppelin.flink.hive.database", "default") - val hiveVersion = properties.getProperty("zeppelin.flink.hive.version", "2.3.4") + val hiveVersion = properties.getProperty("zeppelin.flink.hive.version", "2.3.7") val hiveCatalog = new HiveCatalog("hive", database, hiveConfDir.toString, hiveVersion) this.btenv.registerCatalog("hive", hiveCatalog) this.btenv.useCatalog("hive") diff --git a/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java b/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java index e51042dc14b..29940d46717 100644 --- a/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java +++ b/flink/flink-scala-2.12/src/test/java/org/apache/zeppelin/flink/FlinkSqlInterpreterTest.java @@ -90,7 +90,7 @@ protected Properties getFlinkProperties() throws IOException { p.setProperty("zeppelin.flink.enableHive", "true"); p.setProperty("taskmanager.managed.memory.size", "32"); p.setProperty("taskmanager.memory.task.off-heap.size", "80mb"); - p.setProperty("zeppelin.flink.hive.version", "2.3.4"); + p.setProperty("zeppelin.flink.hive.version", "2.3.7"); p.setProperty("zeppelin.pyflink.useIPython", "false"); p.setProperty("local.number-taskmanager", "4"); p.setProperty("zeppelin.python.gatewayserver_address", "127.0.0.1"); diff --git a/flink/flink1.15-shims/pom.xml b/flink/flink1.15-shims/pom.xml index d6affc169d4..8d549134c90 100644 --- a/flink/flink1.15-shims/pom.xml +++ b/flink/flink1.15-shims/pom.xml @@ -26,9 +26,7 @@ 4.0.0 - org.apache.zeppelin flink1.15-shims - 0.12.0-SNAPSHOT jar Zeppelin: Flink1.15 Shims diff --git a/flink/flink1.16-shims/pom.xml b/flink/flink1.16-shims/pom.xml index 83fe36ab2f6..2bd14cfc5e7 100644 --- a/flink/flink1.16-shims/pom.xml +++ b/flink/flink1.16-shims/pom.xml @@ -26,9 +26,7 @@ 4.0.0 - org.apache.zeppelin flink1.16-shims - 0.12.0-SNAPSHOT jar Zeppelin: Flink1.16 Shims diff --git a/hbase/pom.xml b/hbase/pom.xml index 46a2e63dd42..e7c623346c7 100644 --- a/hbase/pom.xml +++ b/hbase/pom.xml @@ -33,56 +33,15 @@ hbase - 2.4.12 - ${hadoop2.7.version} 1.6.8 - 2.5.0 - 2.12.1 - - - - org.apache.commons - commons-exec - ${commons.exec.version} - org.jruby jruby-complete ${jruby.version} - - org.apache.hadoop - hadoop-yarn-common - ${hbase.hadoop.version} - - - org.apache.hadoop - hadoop-yarn-api - ${hbase.hadoop.version} - - - org.apache.hbase - hbase-client - ${hbase.hbase.version} - - - org.apache.hbase - hbase-annotations - ${hbase.hbase.version} - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - jline - jline - ${jline.version} - diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 4cc79044c01..35be95c71d7 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -35,7 +35,7 @@ jdbc 42.7.2 - ${hadoop3.1.version} + ${hadoop3.3.version} 2.2.220 2.0.1 3.1.3 @@ -85,65 +85,11 @@ org.apache.hadoop - hadoop-client + hadoop-client-runtime ${hadoop.version} provided - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-server - - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.apache.zookeeper - zookeeper - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - - org.apache.hive hive-jdbc @@ -170,21 +116,25 @@ org.apache.hbase hbase-server + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-mapreduce + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + + + hbase-hadoop2-compat + org.apache.hbase + - - org.apache.httpcomponents - httpcore - 4.4.1 - provided - - - org.apache.httpcomponents - httpclient - 4.5.13 - - net.jodah concurrentunit @@ -233,128 +183,6 @@ - - - jdbc-hadoop2 - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop-common.version} - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-server - - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.apache.zookeeper - zookeeper - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - - - - - - jdbc-hadoop3 - - ${hadoop3.0.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop-common.version} - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-server - - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.apache.zookeeper - zookeeper - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - - - diff --git a/livy/pom.xml b/livy/pom.xml index 8ad4917abf0..e004b8424eb 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -41,7 +41,7 @@ 0.7.1-incubating 2.4.8 - ${hadoop2.7.version} + ${hadoop3.3.version} @@ -80,28 +80,24 @@ spark-yarn_${scala.binary.version} + hadoop-client org.apache.hadoop - hadoop-auth - org.apache.hadoop hadoop-common - - org.apache.hadoop - hadoop-hdfs + hadoop-hdfs org.apache.hadoop - hadoop-yarn-client + hadoop-yarn-client org.apache.hadoop - hadoop-client - org.apache.hadoop hadoop-yarn-server-tests + org.apache.hadoop @@ -177,125 +173,25 @@ org.apache.hadoop - hadoop-auth - ${hadoop.version} - test - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - test - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-common - tests - ${hadoop.version} - test - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - - - io.netty - netty - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-hdfs - tests + hadoop-client-api ${hadoop.version} test - - - io.netty - netty - - - com.google.guava - guava - - org.apache.hadoop - hadoop-client + hadoop-client-runtime ${hadoop.version} test - - - com.google.guava - guava - - org.apache.hadoop - hadoop-yarn-client + hadoop-client-minicluster ${hadoop.version} test - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - test - - - com.google.guava - guava - - - - org.apache.hadoop - hadoop-yarn-server-tests - tests - ${hadoop.version} - test - - - com.google.guava - guava - - - diff --git a/pom.xml b/pom.xml index 5dd9007c530..f840d7915b5 100644 --- a/pom.xml +++ b/pom.xml @@ -101,8 +101,8 @@ ${java.version} ${java.version} - ${scala.2.11.version} - 2.11 + ${scala.2.12.version} + 2.12 2.11.12 2.12.17 3.2.15 @@ -139,17 +139,12 @@ 3.6.3 4.1.14 1.6.0 + 3.0.2 - 2.7.7 - 3.0.3 - 3.1.3 3.2.4 3.3.6 - ${hadoop2.7.version} + ${hadoop3.3.version} provided - hadoop-client - hadoop-yarn-api - hadoop-client 2.3.2 1.5.4 @@ -339,14 +334,6 @@ ${commons.configuration2.version} - - - commons-lang - commons-lang - 2.6 - - commons-codec commons-codec @@ -371,6 +358,13 @@ ${commons.cli.version} + + + com.google.code.findbugs + jsr305 + ${findbugs.jsr305.version} + + org.apache.shiro @@ -383,770 +377,28 @@ ${shiro.version} - org.apache.shiro - shiro-config-core - ${shiro.version} - - - - org.bouncycastle - bcpkix-jdk15on - ${bouncycastle.version} - - - - org.codehaus.jettison - jettison - ${jettison.version} - - - - org.apache.hadoop - ${hadoop-client-api.artifact} - ${hadoop.version} - ${hadoop.deps.scope} - - - org.apache.zookeeper - zookeeper - - - org.apache.hadoop - hadoop-common - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - io.netty - netty-all - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.apache.commons - commons-math3 - - - com.fasterxml.jackson.core - jackson-annotations - - - com.nimbusds - nimbus-jose-jwt - - - org.eclipse.jetty - jetty-xml - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-util - - - commons-beanutils - commons-beanutils - - - org.apache.commons - commons-configuration2 - - - commons-beanutils - commons-beanutils-core - - - org.eclipse.jetty - jetty-webapp - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - commons-logging - commons-logging - - - - - - org.apache.hadoop - hadoop-yarn-common - ${hadoop.version} - ${hadoop.deps.scope} - - - asm - asm - - - org.ow2.asm - asm - - - org.jboss.netty - netty - - - javax.servlet - servlet-api - - - commons-logging - commons-logging - - - com.sun.jersey - * - - - com.sun.jersey.jersey-test-framework - * - - - com.sun.jersey.contribs - * - - - com.google.guava - guava - - - org.apache.commons - commons-compress - - - - - - org.apache.hadoop - hadoop-yarn-client - ${hadoop.version} - ${hadoop.deps.scope} - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-core-asl - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.apache.commons - commons-math3 - - - - commons-logging - commons-logging - - - log4j - log4j - - - - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop.version} - ${hadoop.deps.scope} - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-core-asl - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.apache.commons - commons-math3 - - - - commons-logging - commons-logging - - - - - - - - org.apache.hadoop - hadoop-yarn-server-tests - ${hadoop.version} - tests - test - - - org.apache.hadoop - hadoop-yarn-common - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-xc - - - org.codehaus.jackson - jackson-mapper-asl - - - com.google.guava - guava - - - javax.xml.bind - jaxb-api - - - com.fasterxml.jackson.core - jackson-core - - - org.eclipse.jetty - jetty-util - - - com.zaxxer - HikariCP-java7 - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.module - jackson-module-jaxb-annotations - - - - commons-logging - commons-logging - - - io.dropwizard.metrics - metrics-core - - - com.google.guava - guava - - - com.fasterxml.jackson.core - jackson-databind - - + org.apache.shiro + shiro-config-core + ${shiro.version} - org.apache.hadoop - hadoop-common - ${hadoop.version} - ${hadoop.deps.scope} - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-core-asl - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.apache.commons - commons-math3 - - - commons-beanutils - commons-beanutils - - - commons-beanutils - commons-beanutils-core - - - org.apache.commons - commons-configuration2 - - - org.apache.zookeeper - zookeeper - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-util - - - org.eclipse.jetty - jetty-webapp - - - org.eclipse.jetty - jetty-server - - - com.nimbusds - nimbus-jose-jwt - - - com.fasterxml.jackson.core - jackson-databind - - - - commons-logging - commons-logging - - - org.ow2.asm - asm - - - com.jamesmurty.utils - java-xmlbuilder - - + org.bouncycastle + bcpkix-jdk15on + ${bouncycastle.version} + + + + org.codehaus.jettison + jettison + ${jettison.version} org.apache.hadoop - hadoop-common + hadoop-client-api ${hadoop.version} - tests - test - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-server - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-core-asl - - - com.google.guava - guava - - - com.google.code.findbugs - jsr305 - - - org.apache.commons - commons-math3 - - - commons-beanutils - commons-beanutils - - - org.apache.commons - commons-configuration2 - - - org.apache.zookeeper - zookeeper - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-util - - - org.eclipse.jetty - jetty-webapp - - - org.eclipse.jetty - jetty-server - - - com.nimbusds - nimbus-jose-jwt - - - com.fasterxml.jackson.core - jackson-databind - - - - commons-logging - commons-logging - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.ow2.asm - asm - - + ${hadoop.deps.scope} @@ -1208,196 +460,22 @@ org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - com.google.guava - guava - - - io.netty - netty-all - - - org.eclipse.jetty - jetty-util - - - com.fasterxml.jackson.core - jackson-annotations - - - - commons-logging - commons-logging - - - com.fasterxml.jackson.core - jackson-databind - - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - tests - test - - - com.sun.jersey - jersey-json - - - com.sun.jersey - jersey-client - - - javax.servlet - servlet-api - - - org.apache.avro - avro - - - org.apache.jackrabbit - jackrabbit-webdav - - - io.netty - netty - - - commons-httpclient - commons-httpclient - - - org.eclipse.jgit - org.eclipse.jgit - - - com.jcraft - jsch - - - org.apache.commons - commons-compress - - - xml-apis - xml-apis - - - xerces - xercesImpl - - - com.google.guava - guava - - - io.netty - netty-all - - - org.eclipse.jetty - jetty-util - - - com.fasterxml.jackson.core - jackson-annotations - - - - commons-logging - commons-logging - - - log4j - log4j - - - com.fasterxml.jackson.core - jackson-databind - - - - - - org.apache.hadoop - ${hadoop-client-runtime.artifact} + hadoop-client-runtime ${hadoop.version} ${hadoop.deps.scope} - commons-logging commons-logging - - - com.google.code.findbugs - jsr305 + commons-logging org.apache.hadoop - ${hadoop-client-minicluster.artifact} + hadoop-client-minicluster ${hadoop.version} test - diff --git a/rlang/pom.xml b/rlang/pom.xml index 5a7099d13c0..484380de799 100644 --- a/rlang/pom.xml +++ b/rlang/pom.xml @@ -105,12 +105,6 @@ org.apache.spark spark-core_2.12 ${spark.version} - - - org.apache.hadoop - hadoop-client-api - - @@ -121,22 +115,12 @@ org.apache.hadoop - hadoop-client - ${hadoop.version} - compile + hadoop-client-api org.apache.hadoop - hadoop-common - ${hadoop.version} - compile - - - log4j - log4j - - + hadoop-client-runtime @@ -145,7 +129,6 @@ 1.4.9 test - diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index a977ab6978f..3156a170537 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -166,36 +166,6 @@ spark-core_${spark.scala.binary.version} ${spark.version} provided - - - org.apache.hadoop - hadoop-client - - - - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - provided - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - - com.google.protobuf - protobuf-java - - - commons-lang - commons-lang - - diff --git a/spark/spark-scala-parent/pom.xml b/spark/spark-scala-parent/pom.xml index adff9ad2048..cc6fef85885 100644 --- a/spark/spark-scala-parent/pom.xml +++ b/spark/spark-scala-parent/pom.xml @@ -80,21 +80,6 @@ provided - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - provided - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - - org.scala-lang scala-compiler diff --git a/spark/spark-shims/pom.xml b/spark/spark-shims/pom.xml index 10a2231c2a2..80112b78012 100644 --- a/spark/spark-shims/pom.xml +++ b/spark/spark-shims/pom.xml @@ -38,8 +38,7 @@ --> org.apache.hadoop - hadoop-common - ${hadoop.version} + hadoop-client-api provided diff --git a/submarine/pom.xml b/submarine/pom.xml index 831f728204a..e162316d50f 100644 --- a/submarine/pom.xml +++ b/submarine/pom.xml @@ -33,7 +33,7 @@ submarine - ${hadoop2.7.version} + ${hadoop3.3.version} 2.5.4 0.3.8 24.1.1-jre @@ -79,57 +79,8 @@ org.apache.hadoop - hadoop-common + hadoop-client-runtime ${hadoop.version} - - - org.apache.commons - commons-compress - - - com.google.guava - guava - - - org.codehaus.jackson - jackson-mapper-asl - - - org.codehaus.jackson - jackson-xc - - - org.codehaus.jackson - jackson-jaxrs - - - org.codehaus.jackson - jackson-core-asl - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - - - com.google.guava - guava - - - io.netty - netty - - com.hubspot.jinjava diff --git a/zeppelin-interpreter-integration/pom.xml b/zeppelin-interpreter-integration/pom.xml index 85234ebad6e..c79306eb3ff 100644 --- a/zeppelin-interpreter-integration/pom.xml +++ b/zeppelin-interpreter-integration/pom.xml @@ -58,10 +58,6 @@ zeppelin-zengine ${project.version} - - com.google.guava - guava - org.ow2.asm asm @@ -69,12 +65,6 @@ - - com.google.guava - guava - 20.0 - - org.apache.zeppelin zeppelin-server @@ -99,12 +89,6 @@ ${project.version} tests test - - - com.google.guava - guava - - @@ -178,21 +162,19 @@ ${hadoop3.3.version} - hadoop-client-runtime - hadoop-client-minicluster org.apache.hadoop - ${hadoop-client-runtime.artifact} + hadoop-client-runtime test org.apache.hadoop - ${hadoop-client-minicluster.artifact} + hadoop-client-minicluster test diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java index 72dff00c9e4..0d8167ab785 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/FlinkIntegrationTest.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.integration; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -55,7 +56,6 @@ public abstract class FlinkIntegrationTest { private static InterpreterSettingManager interpreterSettingManager; private String flinkVersion; - private String scalaVersion; private String hadoopHome; private String flinkHome; @@ -63,15 +63,15 @@ public void download(String flinkVersion, String scalaVersion) throws IOExceptio LOGGER.info("Testing FlinkVersion: " + flinkVersion); LOGGER.info("Testing ScalaVersion: " + scalaVersion); this.flinkVersion = flinkVersion; - this.scalaVersion = scalaVersion; this.flinkHome = DownloadUtils.downloadFlink(flinkVersion, scalaVersion); - this.hadoopHome = DownloadUtils.downloadHadoop("2.7.7"); + this.hadoopHome = DownloadUtils.downloadHadoop("3.3.6"); } @BeforeAll public static void setUp() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/hadoop-minicluster"); hadoopCluster = new MiniHadoopCluster(conf); hadoopCluster.start(); @@ -141,7 +141,6 @@ public void testLocalMode() throws IOException, YarnException, InterpreterExcept interpreterSettingManager.close(); } - // TODO(zjffdu) enable it when make yarn integration test work @Test public void testYarnMode() throws IOException, InterpreterException, YarnException { InterpreterSetting flinkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("flink"); diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java index dfd9be366da..27c511e64e1 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest32.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.integration; -import org.apache.zeppelin.interpreter.InterpreterSetting; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -25,16 +24,6 @@ public class SparkIntegrationTest32 { - @Nested - @DisplayName("Hadoop2") - public class Hadoop2 extends SparkIntegrationTest { - - @BeforeEach - public void downloadSpark() throws IOException { - prepareSpark("3.2.0", "2.7"); - } - } - @Nested @DisplayName("Hadoop3") public class Hadoop3 extends SparkIntegrationTest { diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java index 73846fcf647..9183257184c 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest33.java @@ -24,16 +24,6 @@ public class SparkIntegrationTest33 { - @Nested - @DisplayName("Hadoop2") - public class Hadoop2 extends SparkIntegrationTest { - - @BeforeEach - public void downloadSpark() throws IOException { - prepareSpark("3.3.0", "2"); - } - } - @Nested @DisplayName("Hadoop3") public class Hadoop3 extends SparkIntegrationTest { diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java index 8af3ee6d5e7..d66bdad0536 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java @@ -22,17 +22,7 @@ import org.junit.jupiter.api.Nested; import java.io.IOException; -public class SparkIntegrationTest34 extends SparkIntegrationTest { - - @Nested - @DisplayName("Hadoop2") - public class Hadoop2 extends SparkIntegrationTest { - - @BeforeEach - public void downloadSpark() throws IOException { - prepareSpark("3.4.0", "2"); - } - } +public class SparkIntegrationTest34 { @Nested @DisplayName("Hadoop3") diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java index ac1952494d4..c14e002650d 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinFlinkClusterTest.java @@ -45,11 +45,9 @@ public abstract class ZeppelinFlinkClusterTest extends AbstractTestRestApi { private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinFlinkClusterTest.class); - private String flinkVersion; private String flinkHome; public void download(String flinkVersion, String scalaVersion) { - this.flinkVersion = flinkVersion; LOGGER.info("Testing FlinkVersion: " + flinkVersion); LOGGER.info("Testing ScalaVersion: " + scalaVersion); this.flinkHome = DownloadUtils.downloadFlink(flinkVersion, scalaVersion); diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java index 18d5b701777..1f1b7692450 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest32.java @@ -23,16 +23,6 @@ public class ZeppelinSparkClusterTest32 { - @Nested - @DisplayName("Hadoop2") - public class Hadoop2 extends ZeppelinSparkClusterTest { - - @BeforeEach - public void downloadSpark() throws Exception { - prepareSpark("3.2.0", "2.7"); - } - } - @Nested @DisplayName("Hadoop3") public class Hadoop3 extends ZeppelinSparkClusterTest { diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java index dbfacd9e400..43eb620f2d6 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest33.java @@ -23,16 +23,6 @@ public class ZeppelinSparkClusterTest33 { - @Nested - @DisplayName("Hadoop2") - public class Hadoop2 extends ZeppelinSparkClusterTest { - - @BeforeEach - public void downloadSpark() throws Exception { - prepareSpark("3.3.0", "2"); - } - } - @Nested @DisplayName("Hadoop3") public class Hadoop3 extends ZeppelinSparkClusterTest { diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index 859edab89dc..91c60860c5e 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -253,74 +253,19 @@ - hadoop2 - + hadoop3 true - ${hadoop2.7.version} - hadoop-client - hadoop-yarn-api - hadoop-client + ${hadoop3.3.version} - - org.apache.hadoop - hadoop-common - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hadoop - hadoop-yarn-client - - - log4j - log4j - - - - - - - - hadoop3 - - - ${hadoop3.2.version} - hadoop-client-api - hadoop-client-runtime - hadoop-client-minicluster - - - - - - org.apache.hadoop - ${hadoop-client-runtime.artifact} - org.apache.hadoop - ${hadoop-client-minicluster.artifact} - test - - - junit - junit - - + hadoop-client-runtime diff --git a/zeppelin-plugins/launcher/yarn/pom.xml b/zeppelin-plugins/launcher/yarn/pom.xml index bbb9925114b..5f2dcfc3630 100644 --- a/zeppelin-plugins/launcher/yarn/pom.xml +++ b/zeppelin-plugins/launcher/yarn/pom.xml @@ -73,44 +73,18 @@ - hadoop2 - + hadoop3 true - - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-common - provided - ${hadoop.version} - - - - org.apache.hadoop - hadoop-client - provided - ${hadoop.version} - - - - - - hadoop3 - - ${hadoop3.2.version} - hadoop-client-runtime + ${hadoop3.3.version} org.apache.hadoop - hadoop-client + hadoop-client-api provided ${hadoop.version} diff --git a/zeppelin-plugins/notebookrepo/filesystem/pom.xml b/zeppelin-plugins/notebookrepo/filesystem/pom.xml index c7a07db5192..6e8858802b6 100644 --- a/zeppelin-plugins/notebookrepo/filesystem/pom.xml +++ b/zeppelin-plugins/notebookrepo/filesystem/pom.xml @@ -49,42 +49,15 @@ - hadoop2 + hadoop3 true - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-common - provided - ${hadoop.version} - - - org.apache.hadoop - hadoop-client - provided - ${hadoop.version} - - - - - - hadoop3 - - ${hadoop3.2.version} + ${hadoop3.3.version} - - org.apache.hadoop - hadoop-client - provided - ${hadoop.version} - org.apache.hadoop hadoop-client-runtime @@ -100,85 +73,10 @@ - - hadoop2-azure - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-azure - ${hadoop.version} - - - com.fasterxml.jackson.core - jackson-core - - - com.google.guava - guava - - - org.apache.commons - commons-lang3 - - - com.jcraf - jsch - - - org.apache.commons - commons-compress - - - - - com.microsoft.azure - azure-data-lake-store-sdk - ${adl.sdk.version} - - - com.fasterxml.jackson.core - jackson-core - - - - - - - - hadoop2-aws - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-aws - ${hadoop.version} - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - - - hadoop3-azure - ${hadoop3.0.version} + ${hadoop3.3.version} @@ -257,7 +155,7 @@ hadoop3-aws - ${hadoop3.0.version} + ${hadoop3.3.version} diff --git a/zeppelin-plugins/notebookrepo/s3/pom.xml b/zeppelin-plugins/notebookrepo/s3/pom.xml index d6f8c82e970..1df29eee335 100644 --- a/zeppelin-plugins/notebookrepo/s3/pom.xml +++ b/zeppelin-plugins/notebookrepo/s3/pom.xml @@ -66,30 +66,6 @@ - - - com.google.inject - guice - 5.0.1 - - - - org.apache.hadoop - hadoop-yarn-client - ${hadoop2.7.version} - - - javax.xml.bind - jaxb-api - - - diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 2652f19e9f7..41c0ae198c6 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -417,7 +417,7 @@ maven-surefire-plugin - + 1 false -Xmx3g -Xms1g -Dfile.encoding=UTF-8 @@ -475,29 +475,13 @@ - - hadoop2 - - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - ${hadoop.deps.scope} - - - - hadoop3 true - ${hadoop3.2.version} + ${hadoop3.3.version} hadoop-client-api hadoop-client-runtime diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 6bf1a7414d3..cabe6442e1b 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -38,10 +38,22 @@ 0.9.8 1.4.01 2.6.0 + + 2.9.8 4.5.4.201711221230-r 1.6 + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.annocations.version} + + + + ${project.groupId} @@ -198,10 +210,7 @@ commons-vfs2 ${commons.vfs2.version} - - org.codehaus.plexus - plexus-utils - + org.apache.hadoop hadoop-hdfs-client @@ -308,62 +317,18 @@ - hadoop2 - + hadoop3 true - - ${hadoop2.7.version} - - - - org.apache.hadoop - hadoop-common - - - log4j - log4j - - - - - - org.apache.hadoop - hadoop-yarn-client - - - - - - hadoop3 - - - ${hadoop3.2.version} - hadoop-client-api - hadoop-client-runtime - hadoop-client-minicluster - + ${hadoop3.3.version} org.apache.hadoop - ${hadoop-client-runtime.artifact} - - - - org.apache.hadoop - ${hadoop-client-minicluster.artifact} - test - ${hadoop.version} - - - junit - junit - - + hadoop-client-runtime diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java index 8cbbdacd279..6310c1d60a2 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/integration/DownloadUtils.java @@ -78,10 +78,13 @@ public static String downloadFlink(String flinkVersion, String scalaVersion) { + flinkVersion + "/flink-hadoop-compatibility_" + scalaVersion + "-" + flinkVersion + ".jar", "-P", targetFlinkHomeFolder + "/lib"}); runShellCommand(new String[]{"wget", - "https://repo1.maven.org/maven2/org/apache/hive/hive-exec/2.3.4/hive-exec-2.3.4.jar", + "https://repo1.maven.org/maven2/org/apache/hive/hive-exec/2.3.7/hive-exec-2.3.7.jar", "-P", targetFlinkHomeFolder + "/lib"}); runShellCommand(new String[]{"wget", - "https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop2-uber/2.7.5-1.8.1/flink-shaded-hadoop2-uber-2.7.5-1.8.1.jar", + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/3.3.6/hadoop-client-api-3.3.6.jar", + "-P", targetFlinkHomeFolder + "/lib"}); + runShellCommand(new String[]{"wget", + "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/3.3.6/hadoop-client-runtime-3.3.6.jar", "-P", targetFlinkHomeFolder + "/lib"}); runShellCommand(new String[]{"wget", "https://repo1.maven.org/maven2/org/apache/flink/flink-table-api-scala_" + scalaVersion + "/"