Skip to content

Commit

Permalink
formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
dannymeijer committed Jun 4, 2024
1 parent 969c901 commit c06af46
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions src/koheesio/spark/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,12 +188,16 @@ def import_pandas_based_on_pyspark_version():
"""
try:
import pandas as pd

pyspark_version = get_spark_minor_version()
pandas_version = pd.__version__

if (pyspark_version < 3.4 and pandas_version >= '2') or (pyspark_version >= 3.4 and pandas_version < '2'):
raise ImportError(f"For PySpark {pyspark_version}, please install Pandas version {'< 2' if pyspark_version < 3.4 else '>= 2'}")
if (pyspark_version < 3.4 and pandas_version >= "2") or (pyspark_version >= 3.4 and pandas_version < "2"):
raise ImportError(
f"For PySpark {pyspark_version}, "
f"please install Pandas version {'< 2' if pyspark_version < 3.4 else '>= 2'}"
)

return pd
except ImportError:
raise ImportError("Pandas module is not installed.")
except ImportError as e:
raise ImportError("Pandas module is not installed.") from e

0 comments on commit c06af46

Please sign in to comment.