[DOP-9653] Test SparkLocalFS on Spark 3.5

MobileTeleSystems · Oct 6, 2023 · 0191cfe · 0191cfe
1 parent 32c37ed
commit 0191cfe
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 3 deletions.
diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -16,12 +16,18 @@ min_excel: &min_excel
   java-version: 8
   os: ubuntu-latest
 
-max: &max
+max_excel: &max_excel
   spark-version: 3.4.1
   python-version: '3.11'
   java-version: 20
   os: ubuntu-latest
 
+max: &max
+  spark-version: 3.5.0
+  python-version: '3.11'
+  java-version: 20
+  os: ubuntu-latest
+
 latest: &latest
   spark-version: latest
   python-version: '3.11'
@@ -30,13 +36,13 @@ latest: &latest
 
 matrix:
   small:
+  - <<: *max_excel
   - <<: *max
-  - <<: *min_avro
-  - <<: *min_excel
   full:
   - <<: *min
   - <<: *min_avro
   - <<: *min_excel
+  - <<: *max_excel
   - <<: *max
   nightly:
   - <<: *min

diff --git a/tests/.coveragerc b/tests/.coveragerc
@@ -19,3 +19,4 @@ exclude_lines =
     if pyspark_version
     spark = SparkSession._instantiatedSession
     if log.isEnabledFor(logging.DEBUG):
+    if sys.version_info
diff --git a/tests/tests_integration/test_file_format_integration/test_excel_integration.py b/tests/tests_integration/test_file_format_integration/test_excel_integration.py
@@ -33,6 +33,8 @@ def test_excel_reader_with_infer_schema(
     spark_version = get_spark_version(spark)
     if spark_version < (3, 2):
         pytest.skip("Excel files are supported on Spark 3.2+ only")
+    if spark_version >= (3, 5):
+        pytest.skip("Excel files are not supported on Spark 3.5+ yet")
 
     file_df_connection, source_path, _ = local_fs_file_df_connection_with_path_and_files
     df = file_df_dataframe
@@ -81,6 +83,8 @@ def test_excel_reader_with_options(
     spark_version = get_spark_version(spark)
     if spark_version < (3, 2):
         pytest.skip("Excel files are supported on Spark 3.2+ only")
+    if spark_version >= (3, 5):
+        pytest.skip("Excel files are not supported on Spark 3.5+ yet")
 
     local_fs, source_path, _ = local_fs_file_df_connection_with_path_and_files
     df = file_df_dataframe
@@ -117,6 +121,8 @@ def test_excel_writer(
     spark_version = get_spark_version(spark)
     if spark_version < (3, 2):
         pytest.skip("Excel files are supported on Spark 3.2+ only")
+    if spark_version >= (3, 5):
+        pytest.skip("Excel files are not supported on Spark 3.5+ yet")
 
     file_df_connection, source_path = local_fs_file_df_connection_with_path
     df = file_df_dataframe