diff --git a/.gitignore b/.gitignore index 17413a26cf9..867c8af50aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # keep the lines below sorted # standard .bloop and version-specific .bloop3xy generated by buildall -gb -.bloop*/ +.bloop* .cache .classpath .DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 150d9e56f39..0fe64e16ddc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,9 +218,21 @@ not clobbered by repeated `bloopInstall` Maven plugin invocations, and it uses [jq](https://stedolan.github.io/jq/) to post-process JSON-formatted project files such that they compile project classes into non-overlapping set of output directories. +To activate the Spark dependency version 3XY you currently are working with update +the symlink `.bloop` to point to the corresponding directory `.bloop-spark3XY` + +Example usage: +```Bash +./build/buildall --generate-bloop --profile=311,330 +rm -vf .bloop +ln -s .bloop-spark330 .bloop +``` + You can now open the spark-rapids as a [BSP project in IDEA](https://www.jetbrains.com/help/idea/bsp-support.html) +Read on for VS Code Scala Metals instructions. + # Bloop, Scala Metals, and Visual Studio Code _Last tested with 1.63.0-insider (Universal) Commit: bedf867b5b02c1c800fbaf4d6ce09cefba_ diff --git a/build/buildall b/build/buildall index 73d39bee94b..fc3ddb85fe5 100755 --- a/build/buildall +++ b/build/buildall @@ -22,6 +22,7 @@ shopt -s extglob BLOOP_VERSION=${BLOOP_VERSION:-"1.4.13"} BLOOP_SCALA_VERSION=${BLOOP_SCALA_VERSION:-"2.13"} SKIP_CLEAN=1 +BUILD_ALL_DEBUG=0 function print_usage() { echo "Usage: buildall [OPTION]" @@ -52,18 +53,20 @@ function print_usage() { } function bloopInstall() { - BLOOP_DIR="${BLOOP_DIR:-$PWD/.bloop}" - mkdir -p $BLOOP_DIR - rm -f $BLOOP_DIR/* - time ( - for bv in $SPARK_SHIM_VERSIONS; do - bloop_config_dir="$PWD/.bloop$bv" - mkdir -p "$bloop_config_dir" - rm -f "$bloop_config_dir"/* + [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x + + local bloopTmpDir=$(mktemp -d /tmp/tmp.bloop.XXXXXX) - $MVN install ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall -pl dist -am \ - -Dbloop.configDirectory="$bloop_config_dir" \ + time ( + bloopDirsGenerated=() + for bv in "${SPARK_SHIM_VERSIONS[@]}"; do + bloopTmpConfigDir="$bloopTmpDir/.bloop$bv" + mkdir -p $bloopTmpConfigDir + $MVN -B clean install \ + ch.epfl.scala:maven-bloop_${BLOOP_SCALA_VERSION}:${BLOOP_VERSION}:bloopInstall \ + -pl aggregator -am \ + -Dbloop.configDirectory="$bloopTmpConfigDir" \ -DdownloadSources=true \ -Dbuildver="$bv" \ -DskipTests \ @@ -73,24 +76,21 @@ function bloopInstall() { -Dmaven.updateconfig.skip=true specifier="spark$bv" - for bloop_json in $(echo $bloop_config_dir/*.json); do - IFS="/" <<< "$bloop_json" read -ra bloop_json_parts - last_idx=$((${#bloop_json_parts[@]} - 1)) - file="${bloop_json_parts[$last_idx]}" - project="${file%.json}-$specifier" - < $bloop_json jq \ - --arg specifier "$specifier" \ - '.project.out=.project.out + "/" + $specifier | .project.name=.project.name + "-" + $specifier' \ - > "$BLOOP_DIR/$project.json" - done + bloopDir=$PWD/.bloop-$specifier + rm -rf $bloopDir + mv $bloopTmpConfigDir $bloopDir + echo "generated bloop files under $bloopDir" + bloopDirsGenerated+=($bloopDir) done - - echo "Generated Bloop files under $BLOOP_DIR" + echo "#### Created bloop projects ${bloopDirsGenerated[@]}" + echo "Execute" + echo " ln -s .bloop-spark3XY .bloop" + echo "to make it an active Bloop project in VS Code Scala Metals" ) } function versionsFromDistProfile() { - [[ BUILD_ALL_DEBUG == "1" ]] && set -x + [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x versionRawStr=$(mvn -B help:evaluate -q -pl dist -P"$1" -Dexpression=included_buildvers -DforceStdout) versionStr=${versionRawStr//[$'\n',]/} echo -n $versionStr @@ -124,8 +124,8 @@ case "$1" in ;; --debug) + BUILD_ALL_DEBUG=1 set -x - BUILD_ALL_DEBUG="1" ;; --clean) @@ -182,6 +182,7 @@ case $DIST_PROFILE in esac echo "Spark versions involved: ${SPARK_SHIM_VERSIONS[@]} ..." +export MVN_BASE_DIR=$($MVN help:evaluate -Dexpression=project.basedir -q -DforceStdout) if [[ "$GEN_BLOOP" == "true" ]]; then bloopInstall @@ -202,10 +203,8 @@ fi echo "Building a combined dist jar with Shims for ${SPARK_SHIM_VERSIONS[@]} ..." -export MVN_BASE_DIR=$($MVN help:evaluate -Dexpression=project.basedir -q -DforceStdout) - function build_single_shim() { - [[ BUILD_ALL_DEBUG == "1" ]] && set -x + [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x BUILD_VER=$1 mkdir -p "$MVN_BASE_DIR/target" (( BUILD_PARALLEL == 1 || NUM_SHIMS == 1 )) && LOG_FILE="/dev/tty" || \