From 3226eae3c5819e551a0659cebea441aaec69bd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Tue, 3 Oct 2023 06:52:41 +0000 Subject: [PATCH] [DOP-9653] Update Excel package version to 0.20.1 --- .github/workflows/data/local-fs/matrix.yml | 2 +- docs/changelog/next_release/159.feature.rst | 2 +- docs/changelog/next_release/161.feature.rst | 1 + onetl/file/format/excel.py | 14 +++++++------- .../test_file/test_format_unit/test_excel_unit.py | 14 +++++++------- 5 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/next_release/161.feature.rst diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml index e956169ba..d491f7da8 100644 --- a/.github/workflows/data/local-fs/matrix.yml +++ b/.github/workflows/data/local-fs/matrix.yml @@ -17,7 +17,7 @@ min_excel: &min_excel os: ubuntu-latest max: &max - spark-version: 3.4.1 + spark-version: 3.5.0 python-version: '3.11' java-version: 20 os: ubuntu-latest diff --git a/docs/changelog/next_release/159.feature.rst b/docs/changelog/next_release/159.feature.rst index 90363f312..a71266620 100644 --- a/docs/changelog/next_release/159.feature.rst +++ b/docs/changelog/next_release/159.feature.rst @@ -1 +1 @@ -Tested compatibility with Spark 3.5.0. ``MongoDB`` and ``Excel`` are not supported yet, but other packages do. +Tested compatibility with Spark 3.5.0. ``MongoDB`` is not supported yet, but other packages do. diff --git a/docs/changelog/next_release/161.feature.rst b/docs/changelog/next_release/161.feature.rst new file mode 100644 index 000000000..533568b69 --- /dev/null +++ b/docs/changelog/next_release/161.feature.rst @@ -0,0 +1 @@ +Update ``Excel`` package version to 0.20.1. diff --git a/onetl/file/format/excel.py b/onetl/file/format/excel.py index ffd11a5da..506f5f4ba 100644 --- a/onetl/file/format/excel.py +++ b/onetl/file/format/excel.py @@ -100,7 +100,7 @@ class Excel(ReadWriteFileFormat): from pyspark.sql import SparkSession # Create Spark session with Excel package loaded - maven_packages = Excel.get_packages(spark_version="3.4.1") + maven_packages = Excel.get_packages(spark_version="3.5.0") spark = ( SparkSession.builder.appName("spark-app-name") .config("spark.jars.packages", ",".join(maven_packages)) @@ -150,7 +150,7 @@ def get_packages( If ``None``, ``spark_version`` is used to determine Scala version. version: str, optional - Package version in format ``major.minor.patch``. Default is ``0.19.0``. + Package version in format ``major.minor.patch``. Default is ``0.20.1``. .. warning:: @@ -168,12 +168,12 @@ def get_packages( from onetl.file.format import Excel - Excel.get_packages(spark_version="3.4.1") - Excel.get_packages(spark_version="3.4.1", scala_version="2.13") + Excel.get_packages(spark_version="3.5.0") + Excel.get_packages(spark_version="3.5.0", scala_version="2.13") Excel.get_packages( - spark_version="3.4.1", + spark_version="3.5.0", scala_version="2.13", - package_version="0.19.0", + package_version="0.20.1", ) """ @@ -187,7 +187,7 @@ def get_packages( raise ValueError(f"Package version should be at least 0.15, got {package_version}") log.warning("Passed custom package version %r, it is not guaranteed to be supported", package_version) else: - version = Version.parse("0.19.0") + version = Version.parse("0.20.1") spark_ver = Version.parse(spark_version) if spark_ver < (3, 2): diff --git a/tests/tests_unit/test_file/test_format_unit/test_excel_unit.py b/tests/tests_unit/test_file/test_format_unit/test_excel_unit.py index e94386120..3f8736e58 100644 --- a/tests/tests_unit/test_file/test_format_unit/test_excel_unit.py +++ b/tests/tests_unit/test_file/test_format_unit/test_excel_unit.py @@ -32,16 +32,16 @@ def test_excel_get_packages_package_version_not_supported(): "spark_version, scala_version, package_version, packages", [ # Detect Scala version by Spark version - ("3.2.4", None, None, ["com.crealytics:spark-excel_2.12:3.2.4_0.19.0"]), - ("3.4.1", None, None, ["com.crealytics:spark-excel_2.12:3.4.1_0.19.0"]), + ("3.2.4", None, None, ["com.crealytics:spark-excel_2.12:3.2.4_0.20.1"]), + ("3.5.0", None, None, ["com.crealytics:spark-excel_2.12:3.5.0_0.20.1"]), # Override Scala version - ("3.2.4", "2.12", None, ["com.crealytics:spark-excel_2.12:3.2.4_0.19.0"]), - ("3.2.4", "2.13", None, ["com.crealytics:spark-excel_2.13:3.2.4_0.19.0"]), - ("3.4.1", "2.12", None, ["com.crealytics:spark-excel_2.12:3.4.1_0.19.0"]), - ("3.4.1", "2.13", None, ["com.crealytics:spark-excel_2.13:3.4.1_0.19.0"]), + ("3.2.4", "2.12", None, ["com.crealytics:spark-excel_2.12:3.2.4_0.20.1"]), + ("3.2.4", "2.13", None, ["com.crealytics:spark-excel_2.13:3.2.4_0.20.1"]), + ("3.5.0", "2.12", None, ["com.crealytics:spark-excel_2.12:3.5.0_0.20.1"]), + ("3.5.0", "2.13", None, ["com.crealytics:spark-excel_2.13:3.5.0_0.20.1"]), # Override package version ("3.2.0", None, "0.16.0", ["com.crealytics:spark-excel_2.12:3.2.0_0.16.0"]), - ("3.4.1", None, "0.18.0", ["com.crealytics:spark-excel_2.12:3.4.1_0.18.0"]), + ("3.5.0", None, "0.18.0", ["com.crealytics:spark-excel_2.12:3.5.0_0.18.0"]), ], ) def test_excel_get_packages(caplog, spark_version, scala_version, package_version, packages):