Test Spark 4.0

MobileTeleSystems · Jul 29, 2024 · dca6397 · dca6397
1 parent 8d81a0e
commit dca6397
Show file tree

Hide file tree

Showing 21 changed files with 206 additions and 101 deletions.
diff --git a/.github/workflows/data/core/matrix.yml b/.github/workflows/data/core/matrix.yml
@@ -1,17 +1,24 @@
-min: &min
+2x: &2x
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -20,6 +27,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *max, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/hdfs/matrix.yml b/.github/workflows/data/hdfs/matrix.yml
@@ -1,19 +1,27 @@
-min: &min
+2x: &2x
   hadoop-version: hadoop2-hdfs
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   hadoop-version: hadoop3-hdfs
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  hadoop-version: hadoop3-hdfs
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   hadoop-version: hadoop3-hdfs
   spark-version: latest
@@ -23,11 +31,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - *max
-  full:
-    - *min
-    - *max
-  nightly:
-    - *min
-    - *latest
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/hive/matrix.yml b/.github/workflows/data/hive/matrix.yml
@@ -1,17 +1,24 @@
-min: &min
+2x: &2x
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -20,6 +27,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/kafka/matrix.yml b/.github/workflows/data/kafka/matrix.yml
@@ -1,5 +1,5 @@
-min: &min
-  # Headers are supported only since 2.x.
+2x: &2x
+  # Headers are supported only since Kafka 2x.
   # Images before 3.2.3 are not creating kafka_jaas.conf properly, and failing to start
   # https://github.com/bitnami/containers/blob/9db9064668365cac89bff58259f63eb78bb97e79/bitnami/kafka/README.md?plain=1#L933
   kafka-version: 3.2.3
@@ -9,14 +9,22 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   kafka-version: 3.7.1
   pydantic-version: 2
   spark-version: 3.5.1
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  kafka-version: 3.7.1
+  pydantic-version: 2
+  spark-version: 4.0.0
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   kafka-version: latest
   pydantic-version: latest
@@ -26,11 +34,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - *max
-  full:
-    - *min
-    - *max
-  nightly:
-    - *min
-    - *latest
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -1,32 +1,41 @@
-min: &min
+23: &23
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-min_avro: &min_avro
+24: &24
+  # Avro supported only since Spark 2.4
   spark-version: 2.4.8
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-min_excel: &min_excel
+32: &32
+  # Excel supported only since Spark 3.2
   spark-version: 3.2.4
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+35: &35
   # Excel package currently has no release for 3.5.1
   spark-version: 3.5.0
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   pydantic-version: latest
@@ -35,15 +44,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small:
-    - <<: *max
-  full:
-    - <<: *min
-    - <<: *min_avro
-    - <<: *min_excel
-    - <<: *max
-  nightly:
-    - <<: *min
-    - <<: *min_avro
-    - <<: *min_excel
-    - <<: *latest
+  small: [*35]
+  full: [*23, *24, *32, *35, *4x]
+  nightly: [*23, *24, *32, *35, *latest]
diff --git a/.github/workflows/data/mongodb/matrix.yml b/.github/workflows/data/mongodb/matrix.yml
@@ -1,20 +1,28 @@
-min: &min
+32: &32
   mongodb-version: 4.0.0
-  # MongoDB connector does not support Spark 2.x
+  # MongoDB connector does not support Spark 2x
   spark-version: 3.2.4
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+35: &35
   mongodb-version: 7.0.12
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  mongodb-version: 7.0.12
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   mongodb-version: latest
   spark-version: latest
@@ -24,6 +32,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*35]
+  full: [*32, *35, *4x]
+  nightly: [*32, *35, *latest]
diff --git a/.github/workflows/data/mssql/matrix.yml b/.github/workflows/data/mssql/matrix.yml
@@ -1,19 +1,27 @@
-min: &min
+2x: &2x
   mssql-version: 2017-GA-ubuntu
   spark-version: 2.3.1
   pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   mssql-version: 2022-CU14-ubuntu-22.04
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  mssql-version: 2022-CU14-ubuntu-22.04
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   mssql-version: latest
   spark-version: latest
@@ -23,6 +31,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/mysql/matrix.yml b/.github/workflows/data/mysql/matrix.yml
@@ -1,4 +1,4 @@
-min: &min
+2x: &2x
   # Tags 5.7.6-5.6.12 cannot be downloaded since Docker v26:
   # "Docker Image Format v1 and Docker Image manifest version 2, schema 1 support is disabled by default"
   mysql-version: 5.7.13
@@ -8,14 +8,22 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   mysql-version: 9.0.1
   spark-version: 3.5.1
   pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  mysql-version: 9.0.1
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   mysql-version: latest
   spark-version: latest
@@ -25,6 +33,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]
diff --git a/.github/workflows/data/oracle/matrix.yml b/.github/workflows/data/oracle/matrix.yml
@@ -1,4 +1,4 @@
-min: &min
+2x: &2x
   oracle-image: gvenzl/oracle-xe
   oracle-version: 11.2.0.2-slim-faststart
   db-name: XE
@@ -8,7 +8,7 @@ min: &min
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+3x: &3x
   oracle-image: gvenzl/oracle-free
   oracle-version: 23.4-slim-faststart
   db-name: FREEPDB1
@@ -18,6 +18,16 @@ max: &max
   java-version: 20
   os: ubuntu-latest
 
+4x: &4x
+  oracle-image: gvenzl/oracle-free
+  oracle-version: 23.4-slim-faststart
+  db-name: FREEPDB1
+  spark-version: 4.0.0
+  pydantic-version: 2
+  python-version: '3.12'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   oracle-image: gvenzl/oracle-free
   oracle-version: slim-faststart
@@ -29,6 +39,6 @@ latest: &latest
   os: ubuntu-latest
 
 matrix:
-  small: [*max]
-  full: [*min, *max]
-  nightly: [*min, *latest]
+  small: [*3x]
+  full: [*2x, *3x, *4x]
+  nightly: [*2x, *3x, *latest]