minor version upgrade

upgrade scala & spark version

circleCI test switch to local mode

remove hack to change system env

AccSuite now execute reliably

improve CI sh

re-enable wget silent mode

upgrade CI scripts, local-cluster mode is enabled again

.circleci/config.yml

@@ -1,14 +1,14 @@
 version: 2.1
 
 executors:
-  openjdk11:
+  main:
     docker:
       - image: cimg/openjdk:11.0
     resource_class: large
 
 jobs:
   build:
-    executor: openjdk11
+    executor: main
     steps:
       - checkout
       - run:
@@ -17,22 +17,22 @@ jobs:
             echo "Starting ..."
       - restore_cache:
           keys:
-            - profile-{{ checksum "./dev/profiles/apache-stable/.common.sh" }}
+            - profile-{{ checksum "./dev/profiles/apache-latest/.common.sh" }}
       - run:
           name: Prepare
           command: |
-            ./dev/CI-apache-stable.sh prepare
+            ./dev/CI-apache-latest.sh prepare
       - save_cache:
-          key: profile-{{ checksum "./dev/profiles/apache-stable/.common.sh" }}
+          key: profile-{{ checksum "./dev/profiles/apache-latest/.common.sh" }}
           paths:
             - ~/.ci
       - run:
           name: Run
           command: |
-            ./dev/CI-apache-stable.sh
+            ./dev/CI-apache-latest.sh
 
 workflows:
-  main-stable: # This is the name of the workflow, feel free to change it to better match your workflow.
+  main:
     # Inside the workflow, you define the jobs you want to run.
     # For more details on extending your workflow, see the configuration docs: https://circleci.com/docs/2.0/configuration-reference/#workflows
     jobs:

.gitattributes

@@ -7,3 +7,4 @@
 *.R text eol=lf
 
 # from apache spark
+*.sh text eol=lf

.scalafix.conf

@@ -15,7 +15,6 @@ RemoveUnused {
   imports = true
   privates = true
   locals = true
-  // the following 2 are not compatible with scala 2.11
   patternvars = true
   params = true
 }

build.gradle.kts

@@ -15,6 +15,7 @@ plugins {
 //    base
     `java-library`
     `java-test-fixtures`
+    `jvm-ecosystem`
 
 //    kotlin("jvm") version "1.6.10" // TODO: remove?
 
@@ -54,6 +55,7 @@ if (sonatypeApiUser.isPresent && sonatypeApiKey.isPresent) {
 allprojects {
 
     apply(plugin = "java-library")
+    apply(plugin = "jvm-ecosystem")
 
     // apply(plugin = "bloop")
     // DO NOT enable! In VSCode it will cause the conflict:

buildSrc/src/main/kotlin/Versions.kt

@@ -29,7 +29,7 @@ class Versions(private val self: Project) {
 
     val sparkV: String = self.properties.get("sparkVersion").toString()
 
-    val tikaV: String = "2.4.1"
+    val tikaV: String = "2.9.0"
 
     val jacksonV: String = "2.12.3"
 }

dev/CI-apache-latest.sh

@@ -4,4 +4,4 @@
 
 CRDIR="$(cd "`dirname "$0"`"; pwd)"
 
-"$CRDIR"/CI/main.sh apache-latest ${@}
+"$CRDIR"/CI/main.sh apache-latest "${@}"

dev/CI-apache-local.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# TODO: this file is merely kept for backward compatibility
+
+CRDIR="$(cd "`dirname "$0"`"; pwd)"
+
+"$CRDIR"/CI/main.sh apache-local "${@}"

dev/CI-apache-stable.sh

@@ -4,4 +4,4 @@
 
 CRDIR="$(cd "`dirname "$0"`"; pwd)"
 
-"$CRDIR"/CI/main.sh apache-stable ${@}
+"$CRDIR"/CI/main.sh apache-stable "${@}"

dev/CI/main.sh

@@ -9,5 +9,5 @@ if [ "${2}" = "prepare" ]; then
   exit 0
 fi
 
-"$FWDIR"/CI/update-submodules.sh && \
+"$FWDIR"/update-submodules.sh && \
   "$FWDIR"/CI/pipeline.sh "-PnotLocal" "${BUILD_PROFILES[@]}"

dev/gradle-versions.sh

@@ -3,6 +3,6 @@
 CRDIR="$(cd "`dirname "$0"`"; pwd)"
 FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 
-${FWDIR}/gradlew wrapper --gradle-version=8.1.1
+${FWDIR}/gradlew wrapper --gradle-version=8.4
 
 ${FWDIR}/gradlew dependencyUpdates "$@"

dev/profiles/apache-latest/.common.sh

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
 
-SPARK_NAME="${SPARK:-spark-3.4.0}"
+SPARK_NAME="${SPARK:-spark-3.5.0}"
 SPARK_DIR_ROOT="$HOME/.ci/spark-dist"
 
-SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala2.13
+export SPARK_SCALA_VERSION="2.13"
+SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}"
 
 SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz"
 
@@ -13,4 +14,4 @@ tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT"
 
 export SPARK_HOME="$SPARK_DIR_ROOT/$SPARK_DIR_NAME"
 
-export BUILD_PROFILES=("-PsparkVersion=3.4.0" "-PscalaVersion=2.13.10")
+export BUILD_PROFILES=("-PsparkVersion=3.5.0")

dev/profiles/apache-local/.common.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+SPARK_NAME="${SPARK:-spark-3.5.0}"
+SPARK_DIR_ROOT="$HOME/.ci/spark-dist"
+
+export SPARK_SCALA_VERSION="2.13"
+SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}"
+
+SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz"
+
+# Download Spark
+wget -N "$SPARK_URL" -P "$SPARK_DIR_ROOT"
+tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT"
+
+export BUILD_PROFILES=("-PsparkVersion=3.5.0")

dev/profiles/apache-stable/.common.sh

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
-SPARK_NAME="${SPARK:-spark-3.4.0}"
+SPARK_NAME="${SPARK:-spark-3.5.0}"
 SPARK_DIR_ROOT="$HOME/.ci/spark-dist"
 
-SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala2.13
+export SPARK_SCALA_VERSION="2.13"
+SPARK_DIR_NAME="$SPARK_NAME"-bin-hadoop3-scala"${SPARK_SCALA_VERSION}"
 
 SPARK_URL="http://archive.apache.org/dist/spark/${SPARK_NAME}/${SPARK_DIR_NAME}.tgz"
 
 # Download Spark
-wget -N "$SPARK_URL" -P "$SPARK_DIR_ROOT"
+wget -N -q "$SPARK_URL" -P "$SPARK_DIR_ROOT"
 tar -xzf "$SPARK_DIR_ROOT/$SPARK_DIR_NAME".tgz -C "$SPARK_DIR_ROOT"
 
 export SPARK_HOME="$SPARK_DIR_ROOT/$SPARK_DIR_NAME"
 
-export BUILD_PROFILES=("-PsparkVersion=3.4.0" "-PscalaVersion=2.13.10")
+export BUILD_PROFILES=("-PsparkVersion=3.5.0")

dev/CI/update-submodules.sh → dev/update-submodules.sh

@@ -3,4 +3,5 @@
 # this is required if any git module has been initialised to another remote repo
 git submodule sync && \
 git submodule foreach git fetch && \
-git submodule update --init --recursive
+git submodule foreach git reset --hard && \
+git submodule update --init --recursive --force

gradle.properties

@@ -2,11 +2,11 @@
 scalaGroup=org.scala-lang
 
 #scalaVersion=2.12.17
-scalaVersion=2.13.10
+scalaVersion=2.13.12
 
 noUav
 
-sparkVersion=3.4.0
+sparkVersion=3.5.0
 
 org.gradle.parallel=true
 org.gradle.caching=true

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -83,7 +83,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -130,26 +131,29 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -198,11 +202,11 @@ fi
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

parent/build.gradle.kts

@@ -4,7 +4,7 @@ val vs = versions()
 plugins {
 
     scala
-    id("io.github.cosmicsilence.scalafix") version "0.1.14"
+    id("io.github.cosmicsilence.scalafix") version "0.1.15"
 }
 
 subprojects {
@@ -181,13 +181,13 @@ subprojects {
         both("org.apache.spark:spark-sql_${vs.scala.binaryV}:${vs.sparkV}")
         both("org.apache.spark:spark-mllib_${vs.scala.binaryV}:${vs.sparkV}")
 
-        api("org.scala-lang.modules:scala-collection-compat_${vs.scala.binaryV}:2.10.0")
+        api("org.scala-lang.modules:scala-collection-compat_${vs.scala.binaryV}:2.11.0")
 
         testRuntimeOnly("org.apache.spark:spark-yarn_${vs.scala.binaryV}:${vs.sparkV}")
 
         testFixturesApi("org.scalatest:scalatest_${vs.scala.binaryV}:${vs.scalaTestV}")
 //        testFixturesApi(project(":repack:scalatest-repack", configuration = "shadow"))
-        testFixturesApi("org.junit.jupiter:junit-jupiter:5.9.2")
+        testFixturesApi("org.junit.jupiter:junit-jupiter:5.10.0")
 
         // TODO: alpha project, switch to mature solution once https://github.com/scalatest/scalatest/issues/1454 is solved
         testRuntimeOnly("co.helmethair:scalatest-junit-runner:0.2.0")
@@ -196,7 +196,7 @@ subprojects {
 
 //        testImplementation("org.scalacheck:scalacheck_${vs.scala.binaryV}:1.17.0")
 
-        testImplementation("com.lihaoyi:fastparse_${vs.scala.binaryV}:2.3.3")
+        testImplementation("com.lihaoyi:fastparse_${vs.scala.binaryV}:3.0.2")
 
         testImplementation("com.vladsch.flexmark:flexmark:0.64.8")
     }

parent/core/src/test/scala/com/tribbloids/spookystuff/metrics/AccSuite.scala

@@ -1,9 +1,10 @@
 package com.tribbloids.spookystuff.metrics
 
-import com.tribbloids.spookystuff.testutils.FunSpecx
+import com.tribbloids.spookystuff.testutils.{FunSpecx, TestHelper}
 import org.apache.spark.util.LongAccumulator
+import org.scalatest.BeforeAndAfterEach
 
-class AccSuite extends FunSpecx {
+class AccSuite extends FunSpecx with BeforeAndAfterEach {
 
   // TODO: it is not working
 //  it("FromType") {
@@ -13,6 +14,10 @@ class AccSuite extends FunSpecx {
 //    assert(acc.value == EventTimeStats.zero)
 //  }
 
+  override protected def beforeEach(): Unit = {
+    TestHelper.TestSparkSession
+  }
+
   it("Simple") {
     Acc.Simple(new LongAccumulator)
   }

parent/mldsl/build.gradle.kts

@@ -3,7 +3,7 @@ val vs = versions()
 
 dependencies {
     api("org.scalameta:ascii-graphs_${vs.scala.binaryV}:0.1.2")
-    api("io.github.classgraph:classgraph:4.8.160")
+    api("io.github.classgraph:classgraph:4.8.163")
 
     api("com.lihaoyi:pprint_${vs.scala.binaryV}:0.8.1")