diff --git a/docs/conf.py b/docs/conf.py index f781dddd..867d4daf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -120,9 +120,6 @@ {"rel": "icon", "href": "icon.svg", "type": "image/svg+xml"}, ] -# TODO: remove after https://github.com/mgeier/sphinx-last-updated-by-git/pull/77 -git_exclude_patterns = ["docs/_static/logo_wide.svg"] - # The master toctree document. master_doc = "index" diff --git a/onetl/connection/db_connection/jdbc_connection/connection.py b/onetl/connection/db_connection/jdbc_connection/connection.py index 32c0b65c..0f3ac024 100644 --- a/onetl/connection/db_connection/jdbc_connection/connection.py +++ b/onetl/connection/db_connection/jdbc_connection/connection.py @@ -90,7 +90,7 @@ def sql( query = clear_statement(query) - log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self.jdbc_dialect) + log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self._get_spark_dialect_name()) log.info("|%s| Executing SQL query (on executor):", self.__class__.__name__) log_lines(log, query) @@ -196,7 +196,7 @@ def get_df_schema( columns: list[str] | None = None, options: JDBCReadOptions | None = None, ) -> StructType: - log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self.jdbc_dialect) + log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self._get_spark_dialect_name()) log.info("|%s| Fetching schema of table %r ...", self.__class__.__name__, source) query = self.dialect.get_sql_query(source, columns=columns, limit=0, compact=True) diff --git a/onetl/connection/db_connection/jdbc_mixin/connection.py b/onetl/connection/db_connection/jdbc_mixin/connection.py index 5d309e58..2f25b5a9 100644 --- a/onetl/connection/db_connection/jdbc_mixin/connection.py +++ b/onetl/connection/db_connection/jdbc_mixin/connection.py @@ -205,7 +205,7 @@ def fetch( query = clear_statement(query) - log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self.jdbc_dialect) + log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self._get_spark_dialect_name()) log.info("|%s| Executing SQL query (on driver):", self.__class__.__name__) log_lines(log, query) @@ -278,7 +278,7 @@ def execute( statement = clear_statement(statement) - log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self.jdbc_dialect) + log.info("|%s| Detected dialect: '%s'", self.__class__.__name__, self._get_spark_dialect_name()) log.info("|%s| Executing statement (on driver):", self.__class__.__name__) log_lines(log, statement) @@ -310,16 +310,6 @@ def execute( log_lines(log, str(metrics)) return df - @property - def jdbc_dialect(self): - """ - Returns the JDBC dialect associated with the connection URL. - """ - jdbc_dialects_package = self.spark._jvm.org.apache.spark.sql.jdbc - dialect = jdbc_dialects_package.JdbcDialects.get(self.jdbc_url).toString() - - return dialect.split("$")[0] if "$" in dialect else dialect - @validator("spark") def _check_java_class_imported(cls, spark): try: @@ -429,6 +419,17 @@ def _get_jdbc_connection(self, options: JDBCFetchOptions | JDBCExecuteOptions): self._last_connection_and_options.data = (new_connection, options) return new_connection + def _get_spark_dialect_name(self) -> str: + """ + Returns the name of the JDBC dialect associated with the connection URL. + """ + dialect = self._get_spark_dialect().toString() + return dialect.split("$")[0] if "$" in dialect else dialect + + def _get_spark_dialect(self): + jdbc_dialects_package = self.spark._jvm.org.apache.spark.sql.jdbc + return jdbc_dialects_package.JdbcDialects.get(self.jdbc_url) + def _close_connections(self): with suppress(Exception): # connection maybe not opened yet @@ -571,9 +572,7 @@ def _resultset_to_dataframe(self, result_set) -> DataFrame: from pyspark.sql import DataFrame # noqa: WPS442 - jdbc_dialects_package = self.spark._jvm.org.apache.spark.sql.jdbc # type: ignore - jdbc_dialect = jdbc_dialects_package.JdbcDialects.get(self.jdbc_url) - + jdbc_dialect = self._get_spark_dialect() jdbc_utils_package = self.spark._jvm.org.apache.spark.sql.execution.datasources.jdbc # type: ignore jdbc_utils = jdbc_utils_package.JdbcUtils diff --git a/requirements/docs.txt b/requirements/docs.txt index be2cd127..87768350 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -9,7 +9,8 @@ sphinx<8 sphinx-copybutton sphinx-design sphinx-favicon -sphinx-last-updated-by-git +# https://github.com/mgeier/sphinx-last-updated-by-git/pull/77 +sphinx-last-updated-by-git>=0.3.8 # TODO: uncomment after https://github.com/zqmillet/sphinx-plantuml/pull/4 # sphinx-plantuml sphinx-tabs