merged with master

.github/workflows/connectors_test.yaml

@@ -1,8 +1,8 @@
-name: "Delta Connectors Tests"
+name: "Delta Connectors"
 on: [push, pull_request]
 jobs:
   build:
-    name: "Run tests"
+    name: "DC: Scala ${{ matrix.scala }}"
     runs-on: ubuntu-20.04
     strategy:
       matrix:

.github/workflows/kernel_test.yaml

@@ -1,7 +1,8 @@
-name: "Delta Kernel Tests"
+name: "Delta Kernel"
 on: [push, pull_request]
 jobs:
   test:
+    name: "DK"
     runs-on: ubuntu-20.04
     env:
       SCALA_VERSION: 2.12.18

.github/workflows/spark_examples_test.yaml

@@ -1,7 +1,8 @@
-name: "Delta Spark Local Publishing and Examples Compilation"
+name: "Delta Spark Publishing and Examples"
 on: [push, pull_request]
 jobs:
   test:
+    name: "DSP&E: Scala ${{ matrix.scala }}"
     runs-on: ubuntu-20.04
     strategy:
       matrix:

.github/workflows/spark_master_test.yaml

@@ -1,7 +1,8 @@
-name: "Delta Spark Master Tests"
+name: "Delta Spark Master"
 on: [push, pull_request]
 jobs:
   test:
+    name: "DSM: Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}"
     runs-on: ubuntu-20.04
     strategy:
       matrix:

.github/workflows/spark_python_test.yaml

@@ -1,7 +1,8 @@
-name: "Delta Spark Python Tests"
+name: "Delta Spark Python"
 on: [push, pull_request]
 jobs:
   test:
+    name: "DSP"
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -60,7 +61,7 @@ jobs:
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.0
+          pipenv run pip install pyspark==3.5.3
           pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
           pipenv run pip install black==23.9.1
           pipenv run pip install importlib_metadata==3.10.0

.github/workflows/spark_test.yaml

@@ -1,7 +1,8 @@
-name: "Delta Spark Tests"
+name: "Delta Spark Latest"
 on: [push, pull_request]
 jobs:
   test:
+    name: "DSL: Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}"
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -64,7 +65,7 @@ jobs:
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.2
+          pipenv run pip install pyspark==3.5.3
           pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
           pipenv run pip install black==23.9.1
           pipenv run pip install importlib_metadata==3.10.0

.github/workflows/unidoc.yaml

@@ -1,12 +1,12 @@
-  name: "Unidoc generation"
+  name: "Unidoc"
   on: [push, pull_request]
   jobs:
     build:
-      name: "Generate unidoc"
+      name: "U: Scala ${{ matrix.scala }}"
       runs-on: ubuntu-20.04
       strategy:
         matrix:
-        # These Scala versions must match those in the build.sbt
+          # These Scala versions must match those in the build.sbt
           scala: [2.13.13, 2.12.18]
       steps:
         - name: install java

Dockerfile

@@ -38,7 +38,7 @@ RUN pip3 install --upgrade pip
 # the`version.sbt` file.
 RUN pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
 
-RUN pip3 install pyspark==3.5.2
+RUN pip3 install pyspark==3.5.3
 
 RUN pip3 install mypy==0.982
 

benchmarks/build.sbt

@@ -20,7 +20,7 @@ scalaVersion := "2.12.18"
 lazy val root = (project in file("."))
   .settings(
     name := "benchmarks",
-    libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.5.2" % "provided",
+    libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.5.3" % "provided",
     libraryDependencies += "com.github.scopt" %% "scopt" % "4.0.1",
     libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.13.1",
 

build.sbt

@@ -52,7 +52,7 @@ val all_scala_versions = Seq(scala212, scala213)
 val default_scala_version = settingKey[String]("Default Scala version")
 Global / default_scala_version := scala212
 
-val LATEST_RELEASED_SPARK_VERSION = "3.5.2"
+val LATEST_RELEASED_SPARK_VERSION = "3.5.3"
 val SPARK_MASTER_VERSION = "4.0.0-SNAPSHOT"
 val sparkVersion = settingKey[String]("Spark version")
 spark / sparkVersion := getSparkVersion()
@@ -176,6 +176,7 @@ def crossSparkSettings(): Seq[Setting[_]] = getSparkVersion() match {
     Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / "scala-spark-3.5",
     Test / unmanagedSourceDirectories += (Test / baseDirectory).value / "src" / "test" / "scala-spark-3.5",
     Antlr4 / antlr4Version := "4.9.3",
+    Test / javaOptions ++= Seq("-Dlog4j.configurationFile=log4j2.properties"),
 
     // Java-/Scala-/Uni-Doc Settings
     scalacOptions ++= Seq(
@@ -204,8 +205,9 @@ def crossSparkSettings(): Seq[Setting[_]] = getSparkVersion() match {
       "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
       "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
       "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
-      "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
-    )
+      "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+      "-Dlog4j.configurationFile=log4j2_spark_master.properties"
+    ),
 
     // Java-/Scala-/Uni-Doc Settings
     // This isn't working yet against Spark Master.

connectors/flink/src/main/java/io/delta/flink/sink/internal/DeltaPartitionComputer.java

@@ -5,6 +5,7 @@
 
 import org.apache.flink.streaming.api.functions.sink.filesystem.BucketAssigner;
 import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.conversion.DateDateConverter;
 import org.apache.flink.table.types.logical.LogicalType;
 import org.apache.flink.table.types.logical.LogicalTypeRoot;
 import org.apache.flink.table.types.logical.RowType;
@@ -99,6 +100,10 @@ public LinkedHashMap<String, String> generatePartitionValues(
                     partitionValues.put(partitionKey, String.valueOf(element.getShort(keyIndex)));
                 } else if (keyType.getTypeRoot() == LogicalTypeRoot.TINYINT) {
                     partitionValues.put(partitionKey, String.valueOf(element.getByte(keyIndex)));
+                } else if (keyType.getTypeRoot() == LogicalTypeRoot.DATE) {
+                    DateDateConverter converter = new DateDateConverter();
+                    String value = String.valueOf(converter.toExternal(element.getInt(keyIndex)));
+                    partitionValues.put(partitionKey, value);
                 } else {
                     throw new RuntimeException("Type not supported " + keyType.getTypeRoot());
                 }