diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 1777c29155b..1609ad16cb4 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -66,11 +66,11 @@ jobs: conda list conda info - name: install application with some interpreter - run: mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B + run: ./mvnw install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B - name: install and test plugins - run: mvn package -DskipRat -pl zeppelin-plugins -amd -B + run: ./mvnw package -DskipRat -pl zeppelin-plugins -amd -B - name: run tests with ${{ matrix.hadoop }} - run: mvn verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false + run: ./mvnw verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false test-interpreter-modules: runs-on: ubuntu-20.04 env: @@ -105,7 +105,7 @@ jobs: run: | R -e "IRkernel::installspec()" - name: verify interpreter - run: mvn verify -DskipRat -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.10 -B + run: ./mvnw verify -DskipRat -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.10 -B test-zeppelin-client-integration-test: runs-on: ubuntu-20.04 steps: @@ -139,10 +139,10 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,flink-cmd,flink/flink-scala-2.11,flink/flink-scala-2.12,jdbc,shell -am - mvn package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B + ./mvnw install -DskipTests -DskipRat -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,flink-cmd,flink/flink-scala-2.11,flink/flink-scala-2.12,jdbc,shell -am + ./mvnw package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B - name: run tests - run: mvn test -DskipRat -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest + run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest test-flink-and-flink-integration-test: runs-on: ubuntu-20.04 strategy: @@ -177,10 +177,10 @@ jobs: auto-activate-base: false - name: install environment run: | - mvn install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B - mvn clean package -pl zeppelin-plugins -amd -DskipTests -B + ./mvnw install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B + ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests - run: mvn test -DskipRat -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -DfailIfNoTests=false -B -Dtest=org.apache.zeppelin.flink.*,FlinkIntegrationTest${{ matrix.flink }},ZeppelinFlinkClusterTest${{ matrix.flink }} + run: ./mvnw test -DskipRat -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -DfailIfNoTests=false -B -Dtest=org.apache.zeppelin.flink.*,FlinkIntegrationTest${{ matrix.flink }},ZeppelinFlinkClusterTest${{ matrix.flink }} run-spark-intergration-test: runs-on: ubuntu-20.04 steps: @@ -214,10 +214,10 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B - mvn clean package -pl zeppelin-plugins -amd -DskipTests -B + ./mvnw install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B + ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests - run: mvn test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkSubmitIntegrationTest,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest30,SparkIntegrationTest31 -DfailIfNoTests=false + run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkSubmitIntegrationTest,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest30,SparkIntegrationTest31 -DfailIfNoTests=false jdbcIntegrationTest-and-unit-test-of-Spark-2-4-with-Scala-2-11: runs-on: ubuntu-20.04 @@ -255,10 +255,10 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B - mvn clean package -pl zeppelin-plugins -amd -DskipTests -B + ./mvnw install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B + ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests - run: mvn test -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B -Dtest=JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false + run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration -B -Dtest=JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false spark-2-4-and-scala-2-12: runs-on: ubuntu-20.04 @@ -293,9 +293,9 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B + ./mvnw install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B - name: run tests - run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false + run: ./mvnw test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-2.4 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false spark-3-0-and-scala-2-12-and-other-interpreter: runs-on: ubuntu-20.04 @@ -334,9 +334,9 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B + ./mvnw install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B - name: run tests with ${{ matrix.python }} - run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false + run: ./mvnw test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false spark-3-1-and-scala-2-12-and-other-interpreter: runs-on: ubuntu-20.04 @@ -374,9 +374,9 @@ jobs: run: | R -e "IRkernel::installspec()" - name: install environment - run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B + run: ./mvnw install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B - name: run tests with ${{ matrix.python }} - run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false + run: ./mvnw test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false test-livy-0-5-with-spark-2-2-0-under-python3: runs-on: ubuntu-20.04 steps: @@ -410,8 +410,8 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn install -DskipTests -DskipRat -pl livy -am -B + ./mvnw install -DskipTests -DskipRat -pl livy -am -B ./testing/downloadSpark.sh "2.2.0" "2.6" ./testing/downloadLivy.sh "0.5.0-incubating" - name: run tests - run: mvn verify -DskipRat -pl livy -am -B + run: ./mvnw verify -DskipRat -pl livy -am -B diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 9fb1f8b539c..9a526ca390d 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -44,9 +44,9 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: Install application - run: mvn -B install -DskipTests -DskipRat -pl ${INTERPRETERS} -Phadoop2 -Pscala-2.11 + run: ./mvnw -B install -DskipTests -DskipRat -pl ${INTERPRETERS} -Phadoop2 -Pscala-2.11 - name: Run headless test - run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" mvn verify -DskipRat -pl zeppelin-web -Phadoop2 -Pscala-2.11 -Pweb-e2e -B + run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw verify -DskipRat -pl zeppelin-web -Phadoop2 -Pscala-2.11 -Pweb-e2e -B run-tests-in-zeppelin-web-angular: runs-on: ubuntu-20.04 steps: @@ -69,7 +69,7 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: Run headless test - run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" mvn package -DskipRat -pl zeppelin-web-angular -Pweb-angular -B + run: xvfb-run --auto-servernum --server-args="-screen 0 1024x768x24" ./mvnw package -DskipRat -pl zeppelin-web-angular -Pweb-angular -B test-selenium-with-spark-module-for-spark-2-3: runs-on: ubuntu-20.04 @@ -107,8 +107,8 @@ jobs: R -e "IRkernel::installspec()" - name: install environment run: | - mvn clean install -DskipTests -DskipRat -pl ${INTERPRETERS} -Pspark-2.3 -Phadoop2 -Phelium-dev -Pexamples -Pintegration -Pspark-scala-2.11 -B - mvn clean package -pl zeppelin-plugins -amd -B + ./mvnw clean install -DskipTests -DskipRat -pl ${INTERPRETERS} -Pspark-2.3 -Phadoop2 -Phelium-dev -Pexamples -Pintegration -Pspark-scala-2.11 -B + ./mvnw clean package -pl zeppelin-plugins -amd -B ./testing/downloadSpark.sh "2.3.2" "2.6" - name: run tests - run: xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" mvn verify -DskipRat -Pspark-2.3 -Phadoop2 -Phelium-dev -Pexamples -Pintegration -Pspark-scala-2.11 -B -pl zeppelin-integration -DfailIfNoTests=false + run: xvfb-run --auto-servernum --server-args="-screen 0 1600x1024x16" ./mvnw verify -DskipRat -Pspark-2.3 -Phadoop2 -Phelium-dev -Pexamples -Pintegration -Pspark-scala-2.11 -B -pl zeppelin-integration -DfailIfNoTests=false diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index b32cb598e86..7e1f498d801 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -27,4 +27,4 @@ jobs: distribution: 'adopt' java-version: 8 - name: Check Rat - run: mvn apache-rat:check -Prat -B + run: ./mvnw apache-rat:check -Prat -B diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 00000000000..b901097f2db --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,117 @@ +/* + * Copyright 2007-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 00000000000..2cc7d4a55c0 Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000000..7e83fe97d7d --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/Dockerfile b/Dockerfile index 28606ea4cfb..3eeba4902a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ WORKDIR /workspace/zeppelin # Allow npm and bower to run with root privileges RUN echo "unsafe-perm=true" > ~/.npmrc && \ echo '{ "allow_root": true }' > ~/.bowerrc && \ - mvn -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular && \ + ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular && \ # Example with doesn't compile all interpreters - # mvn -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular -pl '!groovy,!submarine,!livy,!hbase,!pig,!file,!flink,!ignite,!kylin,!lens' && \ + # ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.0 -Pinclude-hadoop -Phadoop3 -Pspark-scala-2.12 -Pweb-angular -pl '!groovy,!submarine,!livy,!hbase,!pig,!file,!flink,!ignite,!kylin,!lens' && \ mv /workspace/zeppelin/zeppelin-distribution/target/zeppelin-*/zeppelin-* /opt/zeppelin/ && \ # Removing stuff saves time, because docker creates a temporary layer rm -rf ~/.m2 && \ diff --git a/beam/README.md b/beam/README.md index 948c95cfc0f..de39815eeb4 100644 --- a/beam/README.md +++ b/beam/README.md @@ -8,7 +8,7 @@ Current interpreter implementation supports the static repl. It compiles the cod You have to first build the Beam interpreter by enable the **beam** profile as follows: ``` -mvn clean package -Pbeam -DskipTests -Pscala-2.10 +./mvnw clean package -Pbeam -DskipTests -Pscala-2.10 ``` ### Notice diff --git a/bigquery/README.md b/bigquery/README.md index 0dff5feb7c8..024d81167da 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -8,7 +8,7 @@ If you like to run these tests manually, please follow the following steps: * [Create a new project](https://support.google.com/cloud/answer/6251787?hl=en) * [Create a Google Compute Engine instance](https://cloud.google.com/compute/docs/instances/create-start-instance) * Copy the project ID that you created and add it to the property "projectId" in `resources/constants.json` -* Run the command mvn -Dbigquery.text.exclude='' test -pl bigquery -am +* Run the command ./mvnw -Dbigquery.text.exclude='' test -pl bigquery -am # Connection The Interpreter opens a connection with the BigQuery Service using the supplied Google project ID and the compute environment variables. diff --git a/dev/change_zeppelin_version.sh b/dev/change_zeppelin_version.sh index 0097e5b837f..00ac88a1778 100755 --- a/dev/change_zeppelin_version.sh +++ b/dev/change_zeppelin_version.sh @@ -36,13 +36,13 @@ is_dev_version() { else return 1 fi -} +} is_maintenance_version() { local version="$1" if [[ "${version}" == *"SNAPSHOT" ]]; then version = $(echo ${1} | cut -d'-' -f 1) - fi + fi if [[ "${version}" == *".0" ]]; then return 1 else @@ -51,7 +51,7 @@ is_maintenance_version() { } # Change version in pom.xml -mvn versions:set -DnewVersion="${TO_VERSION}" -DgenerateBackupPoms=false > /dev/null 2>&1 +./mvnw versions:set -DnewVersion="${TO_VERSION}" -DgenerateBackupPoms=false > /dev/null 2>&1 # Change version in example and package files sed -i '' 's/-'"${FROM_VERSION}"'.jar",/-'"${TO_VERSION}"'.jar",/g' zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json diff --git a/dev/create_release.sh b/dev/create_release.sh index 536f3a5b96c..03d1d056709 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -65,8 +65,8 @@ function make_binary_release() { cp -r "${WORKING_DIR}/zeppelin" "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" cd "${WORKING_DIR}/zeppelin-${RELEASE_VERSION}-bin-${BIN_RELEASE_NAME}" - echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}" - mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS} + echo "./mvnw clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}" + ./mvnw clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS} if [[ $? -ne 0 ]]; then echo "Build failed. ${BUILD_FLAGS}" exit 1 diff --git a/dev/publish_release.sh b/dev/publish_release.sh index b389fd794e1..4231b73ebdf 100755 --- a/dev/publish_release.sh +++ b/dev/publish_release.sh @@ -84,18 +84,18 @@ function publish_snapshot_to_maven() { tmp_repo="$(mktemp -d /tmp/zeppelin-repo-XXXXX)" - mvn versions:set -DnewVersion=$RELEASE_VERSION + ./mvnw versions:set -DnewVersion=$RELEASE_VERSION tmp_settings="tmp-settings.xml" echo "" > $tmp_settings echo "apache.snapshots.https$ASF_USERID" >> $tmp_settings echo "$ASF_PASSWORD" >> $tmp_settings echo "" >> $tmp_settings - mvn --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \ + ./mvnw --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \ $PUBLISH_PROFILES -Drat.skip=true deploy "${BASEDIR}/change_scala_version.sh" 2.11 - mvn -Pscala-2.11 --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \ + ./mvnw -Pscala-2.11 --settings $tmp_settings -Dmaven.repo.local="${tmp_repo}" -Pbeam -DskipTests \ $PUBLISH_PROFILES -Drat.skip=true clean deploy rm $tmp_settings @@ -106,7 +106,7 @@ function publish_to_maven() { cd "${WORKING_DIR}/zeppelin" # Force release version - mvn versions:set -DnewVersion="${RELEASE_VERSION}" + ./mvnw versions:set -DnewVersion="${RELEASE_VERSION}" # Using Nexus API documented here: # https://support.sonatype.com/hc/en-us/articles/213465868-Uploading-to-a-Staging-Repository-via-REST-API @@ -128,9 +128,9 @@ function publish_to_maven() { rm -rf $HOME/.m2/repository/org/apache/zeppelin # build with scala-2.11 - echo "mvn clean install -DskipTests \ + echo "./mvnw clean install -DskipTests \ ${PUBLISH_PROFILES} ${PROJECT_OPTIONS}" - mvn clean install -DskipTests \ + ./mvnw clean install -DskipTests \ ${PUBLISH_PROFILES} ${PROJECT_OPTIONS} if [[ $? -ne 0 ]]; then echo "Build failed." diff --git a/docs/development/contribution/how_to_contribute_code.md b/docs/development/contribution/how_to_contribute_code.md index 3683428f985..719317775c8 100644 --- a/docs/development/contribution/how_to_contribute_code.md +++ b/docs/development/contribution/how_to_contribute_code.md @@ -70,19 +70,19 @@ Before making a pull request, please take a look [Contribution Guidelines](http: ### Build ```bash -mvn install +./mvnw install ``` To skip test ```bash -mvn install -DskipTests +./mvnw install -DskipTests ``` To build with specific spark / hadoop version ```bash -mvn install -Dspark.version=x.x.x -Dhadoop.version=x.x.x +./mvnw install -Dspark.version=x.x.x -Dhadoop.version=x.x.x ``` For the further @@ -97,7 +97,7 @@ For the further ```bash cd zeppelin-server HADOOP_HOME=YOUR_HADOOP_HOME JAVA_HOME=YOUR_JAVA_HOME \ -mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args="" +./mvnw exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexec.args="" ``` #### Option 2 - Daemon Script @@ -105,7 +105,7 @@ mvn exec:java -Dexec.mainClass="org.apache.zeppelin.server.ZeppelinServer" -Dexe > **Note:** Make sure you first run ```bash -mvn clean install -DskipTests +./mvnw clean install -DskipTests ``` in your zeppelin root directory, otherwise your server build will fail to find the required dependencies in the local repro. @@ -140,14 +140,14 @@ cd /zeppelin-interpreter/src/main/thrift Zeppelin has [set of integration tests](https://github.com/apache/zeppelin/tree/master/zeppelin-server/src/test/java/org/apache/zeppelin/integration) using Selenium. To run these test, first build and run Zeppelin and make sure Zeppelin is running on port 8080. Then you can run test using following command ```bash -TEST_SELENIUM=true mvn test -Dtest=[TEST_NAME] -DfailIfNoTests=false \ +TEST_SELENIUM=true ./mvnw test -Dtest=[TEST_NAME] -DfailIfNoTests=false \ -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' ``` For example, to run [ParagraphActionIT](https://github.com/apache/zeppelin/blob/master/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java), ```bash -TEST_SELENIUM=true mvn test -Dtest=ParagraphActionsIT -DfailIfNoTests=false \ +TEST_SELENIUM=true ./mvnw test -Dtest=ParagraphActionsIT -DfailIfNoTests=false \ -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' ``` diff --git a/docs/development/contribution/useful_developer_tools.md b/docs/development/contribution/useful_developer_tools.md index 17ca40307f5..c628a6c278c 100644 --- a/docs/development/contribution/useful_developer_tools.md +++ b/docs/development/contribution/useful_developer_tools.md @@ -61,28 +61,28 @@ you can use this function like `setjdk 1.8` / `setjdk 1.7` ```bash # build `zeppelin-web` only -mvn clean -pl 'zeppelin-web' package -DskipTests; +./mvnw clean -pl 'zeppelin-web' package -DskipTests; # build `zeppelin-server` and its dependencies only -mvn clean package -pl 'spark,spark-dependencies,python,markdown,zeppelin-server' --am -DskipTests +./mvnw clean package -pl 'spark,spark-dependencies,python,markdown,zeppelin-server' --am -DskipTests # build spark related modules with default profiles: scala 2.10 -mvn clean package -pl 'spark,spark-dependencies,zeppelin-server' --am -DskipTests +./mvnw clean package -pl 'spark,spark-dependencies,zeppelin-server' --am -DskipTests # build spark related modules with profiles: scala 2.11, spark 2.1 hadoop 2.7 ./dev/change_scala_version.sh 2.11 -mvn clean package -Pspark-2.1 -Phadoop-2.7 -Pscala-2.11 \ +./mvnw clean package -Pspark-2.1 -Phadoop-2.7 -Pscala-2.11 \ -pl 'spark,spark-dependencies,zeppelin-server' --am -DskipTests # build `zeppelin-server` and `markdown` with dependencies -mvn clean package -pl 'markdown,zeppelin-server' --am -DskipTests +./mvnw clean package -pl 'markdown,zeppelin-server' --am -DskipTests ``` ### Running Individual Tests ```bash # run the `HeliumBundleFactoryTest` test class -mvn test -pl 'zeppelin-server' --am -DfailIfNoTests=false -Dtest=HeliumBundleFactoryTest +./mvnw test -pl 'zeppelin-server' --am -DfailIfNoTests=false -Dtest=HeliumBundleFactoryTest ``` ### Running Selenium Tests @@ -91,12 +91,12 @@ Make sure that Zeppelin instance is started to execute integration tests (= sele ```bash # run the `SparkParagraphIT` test class -TEST_SELENIUM="true" mvn test -pl 'zeppelin-server' --am \ +TEST_SELENIUM="true" ./mvnw test -pl 'zeppelin-server' --am \ -DfailIfNoTests=false -Dtest=SparkParagraphIT # run the `testSqlSpark` test function only in the `SparkParagraphIT` class # but note that, some test might be dependent on the previous tests -TEST_SELENIUM="true" mvn test -pl 'zeppelin-server' --am \ +TEST_SELENIUM="true" ./mvnw test -pl 'zeppelin-server' --am \ -DfailIfNoTests=false -Dtest=SparkParagraphIT#testSqlSpark ``` diff --git a/docs/interpreter/hbase.md b/docs/interpreter/hbase.md index fd6334acebc..50228407c92 100644 --- a/docs/interpreter/hbase.md +++ b/docs/interpreter/hbase.md @@ -32,14 +32,14 @@ By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1 ```bash # HBase 1.1.4 -mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0 +./mvnw clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0 ``` To work with HBase 1.2.0+, use the following build command: ```bash # HBase 1.2.0 -mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.2.0 -Dhbase.hadoop.version=2.6.0 +./mvnw clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.2.0 -Dhbase.hadoop.version=2.6.0 ``` ## Configuration diff --git a/docs/interpreter/scalding.md b/docs/interpreter/scalding.md index 02c5fb8b31f..1d55e59a38f 100644 --- a/docs/interpreter/scalding.md +++ b/docs/interpreter/scalding.md @@ -29,7 +29,7 @@ limitations under the License. You have to first build the Scalding interpreter by enable the **scalding** profile as follows: ```bash -mvn clean package -Pscalding -DskipTests +./mvnw clean package -Pscalding -DskipTests ``` ## Enabling the Scalding Interpreter diff --git a/docs/quickstart/kubernetes.md b/docs/quickstart/kubernetes.md index 8d311fb4ce6..cc003cd9cf1 100644 --- a/docs/quickstart/kubernetes.md +++ b/docs/quickstart/kubernetes.md @@ -124,7 +124,7 @@ Check [here](https://spark.apache.org/docs/latest/running-on-kubernetes.html) to To build your own Zeppelin image, first build Zeppelin project with `-Pbuild-distr` flag. ``` -$ mvn package -DskipTests -Pbuild-distr +$ ./mvnw package -DskipTests -Pbuild-distr ``` Binary package will be created under `zeppelin-distribution/target` directory. Move created package file under `scripts/docker/zeppelin/bin/` directory. diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md index 7f70c33b5c1..5939ca4835d 100644 --- a/docs/setup/basics/how_to_build.md +++ b/docs/setup/basics/how_to_build.md @@ -61,7 +61,7 @@ git clone https://github.com/apache/zeppelin.git You can build Zeppelin with following maven command: ```bash -mvn clean package -DskipTests [Options] +./mvnw clean package -DskipTests [Options] ``` Check [build-profiles](#build-profiles) section for further build options. @@ -164,28 +164,28 @@ Here are some examples with several options: ```bash # build with spark-3.0, spark-scala-2.12 -mvn clean package -Pspark-3.0 -Pspark-scala-2.12 -DskipTests +./mvnw clean package -Pspark-3.0 -Pspark-scala-2.12 -DskipTests # build with spark-2.4, spark-scala-2.11 -mvn clean package -Pspark-2.4 -Pspark-scala-2.11 -DskipTests +./mvnw clean package -Pspark-2.4 -Pspark-scala-2.11 -DskipTests # build with spark-1.6, spark-scala-2.10 -mvn clean package -Pspark-1.6 -Pspark-scala-2.10 -DskipTests +./mvnw clean package -Pspark-1.6 -Pspark-scala-2.10 -DskipTests # build with CDH -mvn clean package -Pspark-1.6 -Pspark-scala-2.10 -Dhadoop.version=2.6.0-cdh5.5.0 -Pvendor-repo -DskipTests +./mvnw clean package -Pspark-1.6 -Pspark-scala-2.10 -Dhadoop.version=2.6.0-cdh5.5.0 -Pvendor-repo -DskipTests ``` Ignite Interpreter ```bash -mvn clean package -Dignite.version=1.9.0 -DskipTests +./mvnw clean package -Dignite.version=1.9.0 -DskipTests ``` Scalding Interpreter ```bash -mvn clean package -Pscalding -DskipTests +./mvnw clean package -Pscalding -DskipTests ``` ### Optional configurations @@ -239,23 +239,11 @@ sudo apt-get install r-base-dev sudo apt-get install r-cran-evaluate ``` - - -### Install maven - -```bash -wget http://www.eu.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -sudo tar -zxf apache-maven-3.6.3-bin.tar.gz -C /usr/local/ -sudo ln -s /usr/local/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn -``` - _Notes:_ - Ensure node is installed by running `node --version` - - Ensure maven is running version 3.6.3 or higher with `mvn -version` + - Ensure maven is running version 3.6.3 or higher with `./mvnw -version` - Configure maven to use more memory than usual by `export MAVEN_OPTS="-Xmx2g -XX:MaxMetaspaceSize=512m"` - - ## Proxy setting (optional) If you're behind the proxy, you'll need to configure maven and npm to pass through it. @@ -325,13 +313,13 @@ _Notes:_ To package the final distribution including the compressed archive, run: ```sh -mvn clean package -Pbuild-distr +./mvnw clean package -Pbuild-distr ``` To build a distribution with specific profiles, run: ```sh -mvn clean package -Pbuild-distr -Pspark-2.4 +./mvnw clean package -Pbuild-distr -Pspark-2.4 ``` The profiles `-Pspark-2.4` can be adjusted if you wish to build to a specific spark versions. diff --git a/docs/setup/deployment/flink_and_spark_cluster.md b/docs/setup/deployment/flink_and_spark_cluster.md index 8aaa4958a23..90fde4464cd 100644 --- a/docs/setup/deployment/flink_and_spark_cluster.md +++ b/docs/setup/deployment/flink_and_spark_cluster.md @@ -73,34 +73,6 @@ sudo apt-get update sudo apt-get install openjdk-7-jdk openjdk-7-jre-lib ``` -##### Maven 3.1+ -Zeppelin requires maven version 3.x. The version available in the repositories at the time of writing is 2.x, so maven must be installed manually. - -Purge any existing versions of maven. - -```bash -sudo apt-get purge maven maven2 -``` - -Download the maven 3.3.9 binary. - -```bash -wget "http://www.us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz" -``` - -Unarchive the binary and move to the `/usr/local` directory. - -```bash -tar -zxvf apache-maven-3.3.9-bin.tar.gz -sudo mv ./apache-maven-3.3.9 /usr/local -``` - -Create symbolic links in `/usr/bin`. - -```bash -sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/bin/mvn -``` - ### Installing Zeppelin This provides a quick overview of Zeppelin installation from source, however the reader is encouraged to review the [Zeppelin Installation Guide](../../quickstart/install.html) @@ -120,7 +92,7 @@ cd zeppelin Package Zeppelin. ```bash -mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.3 -Pscala-2.10 +./mvnw clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.3 -Pscala-2.10 ``` `-DskipTests` skips build tests- you're not developing (yet), so you don't need to do tests, the clone version *should* build. @@ -139,7 +111,7 @@ As long as you didn't edit any code, it is unlikely the build is failing because - Don't get discouraged. - Scroll up and read through the logs. There will be clues there. -- Retry (that is, run the `mvn clean package -DskipTests -Pspark-1.6` again) +- Retry (that is, run the `./mvnw clean package -DskipTests -Pspark-1.6` again) - If there were clues that a dependency couldn't be downloaded wait a few hours or even days and retry again. Open source software when compiling is trying to download all of the dependencies it needs, if a server is off-line there is nothing you can do but wait for it to come back. - Make sure you followed all of the steps carefully. - Ask the community to help you. Go [here](http://zeppelin.apache.org/community.html) and join the user mailing list. People are there to help you. Make sure to copy and paste the build output (everything that happened in the console) and include that in your message. diff --git a/docs/setup/deployment/virtual_machine.md b/docs/setup/deployment/virtual_machine.md index a50d1a2ba52..9288dfbdfdb 100644 --- a/docs/setup/deployment/virtual_machine.md +++ b/docs/setup/deployment/virtual_machine.md @@ -86,7 +86,6 @@ By default, Vagrant will share your project directory (the directory with the Va Running the following commands in the guest machine should display these expected versions: * `node --version` should report *v0.12.7* -* `mvn --version` should report *Apache Maven 3.3.9* and *Java version: 1.7.0_85* The virtual machine consists of: @@ -108,7 +107,7 @@ This assumes you've already cloned the project either on the host machine in the ```bash cd /zeppelin -mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests +./mvnw clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests ./bin/zeppelin-daemon.sh start ``` diff --git a/influxdb/README.md b/influxdb/README.md index 06ba34f6adf..0f6645b4fb5 100644 --- a/influxdb/README.md +++ b/influxdb/README.md @@ -8,11 +8,11 @@ Provide InfluxDB Interpreter for Zeppelin. ## Build ``` -mvn -pl influxdb -am -DskipTests package +./mvnw -pl influxdb -am -DskipTests package ``` ## Test ``` -mvn -pl influxdb -am -Dtest='org.apache.zeppelin.influxdb.*' -DfailIfNoTests=false test +./mvnw -pl influxdb -am -Dtest='org.apache.zeppelin.influxdb.*' -DfailIfNoTests=false test ``` diff --git a/kotlin/README.md b/kotlin/README.md index d66c5a725f6..765df6aa475 100644 --- a/kotlin/README.md +++ b/kotlin/README.md @@ -56,13 +56,13 @@ and `ZeppelinContext` from `SparkInterpreter` in the same session and binds them ## Testing Kotlin Interpreter and Spark Kotlin Interpreter come with unit tests. They can be run with \ -`mvn clean test` \ +`./mvnw clean test` \ in `$ZEPPELIN_HOME/kotlin` for base Kotlin Interpreter and \ -`mvn -Dtest=KotlinSparkInterpreterTest test` \ +`./mvnw -Dtest=KotlinSparkInterpreterTest test` \ in `$ZEPPELIN_HOME/spark/interpreter` for Spark Kotlin Interpreter. To test manually, build Zeppelin with \ -`mvn clean package -DskipTests` \ +`./mvnw clean package -DskipTests` \ and create a note with `kotlin` interpreter for base or `spark` for Spark. In Spark interpreter, add `%spark.kotlin` in the start of paragraph to use Kotlin Spark Interpreter. diff --git a/livy/README.md b/livy/README.md index 403c0d6fa8a..7bb0241f312 100644 --- a/livy/README.md +++ b/livy/README.md @@ -11,5 +11,5 @@ You can add integration test to [LivyInterpreter.java](https://github.com/apache #!/usr/bin/env bash export LIVY_HOME= export SPARK_HOME= -mvn clean verify -pl livy -DfailIfNoTests=false -DskipRat +./mvnw clean verify -pl livy -DfailIfNoTests=false -DskipRat ``` diff --git a/mvnw b/mvnw new file mode 100755 index 00000000000..41c0f0c23db --- /dev/null +++ b/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 00000000000..86115719e53 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,182 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/python/README.md b/python/README.md index 29dc1f0bce5..bde6e8e6a22 100644 --- a/python/README.md +++ b/python/README.md @@ -11,7 +11,7 @@ Current interpreter implementation spawns new system python process through `Pro To run full suit of tests, including ones that depend on real Python interpreter AND external libraries installed (like Pandas, Pandasql, etc) do ``` -mvn -Dpython.test.exclude='' test -pl python -am +./mvnw -Dpython.test.exclude='' test -pl python -am ``` - **Py4j support** diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java index 95594a8e801..57f9420f523 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java @@ -54,7 +54,7 @@ *

* To run manually on such environment, use: * - * mvn -Dpython.test.exclude='' test -pl python -am + * ./mvnw -Dpython.test.exclude='' test -pl python -am * */ @RunWith(value = Parameterized.class) diff --git a/scio/README.md b/scio/README.md index b95c548b7fd..73d0cb67f1d 100644 --- a/scio/README.md +++ b/scio/README.md @@ -8,11 +8,11 @@ Provide Scio Interpreter for Zeppelin. ## Build ``` -mvn -pl zeppelin-interpreter,zeppelin-display,scio -DskipTests package +./mvnw -pl zeppelin-interpreter,zeppelin-display,scio -DskipTests package ``` ## Test ``` -mvn -pl scio,zeppelin-display,zeppelin-interpreter -Dtest='org.apache.zeppelin.scio.*' -DfailIfNoTests=false test +./mvnw -pl scio,zeppelin-display,zeppelin-interpreter -Dtest='org.apache.zeppelin.scio.*' -DfailIfNoTests=false test ```