From f050ef21643a9c6732a562db6696e34559486242 Mon Sep 17 00:00:00 2001 From: rohansun Date: Mon, 2 Sep 2024 02:24:52 -0400 Subject: [PATCH 1/3] Added default_dialect support to SqlQueriesSource --- .../src/datahub/ingestion/source/sql_queries.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py index 138430a68fcb9d..2fe9ace111d6fb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py @@ -66,6 +66,11 @@ class SqlQueriesSourceConfig(PlatformInstanceConfigMixin, EnvConfigMixin): description="The default schema to use for unqualified table names", default=None, ) + default_dialect: Optional[str] = Field( + description="The SQL dialect to use when parsing queries. Overrides automatic dialect detection.", + default=None, + ) + class SqlQueriesSourceReport(SourceReport): @@ -177,6 +182,7 @@ def _process_query(self, entry: "QueryEntry") -> Iterable[MetadataWorkUnit]: schema_resolver=self.schema_resolver, default_db=self.config.default_db, default_schema=self.config.default_schema, + default_dialect=self.config.default_dialect, ) if result.debug_info.table_error: logger.info(f"Error parsing table lineage, {result.debug_info.table_error}") From 05f4fafdc95caffb71389495655441a690b363a5 Mon Sep 17 00:00:00 2001 From: rohansun <66036924+rohansun@users.noreply.github.com> Date: Mon, 2 Sep 2024 02:29:58 -0400 Subject: [PATCH 2/3] Update sql_queries.py --- metadata-ingestion/src/datahub/ingestion/source/sql_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py index 2fe9ace111d6fb..b38c4080397f08 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py @@ -182,7 +182,7 @@ def _process_query(self, entry: "QueryEntry") -> Iterable[MetadataWorkUnit]: schema_resolver=self.schema_resolver, default_db=self.config.default_db, default_schema=self.config.default_schema, - default_dialect=self.config.default_dialect, + default_dialect=self.config.default_dialect, ) if result.debug_info.table_error: logger.info(f"Error parsing table lineage, {result.debug_info.table_error}") From 9e463dc993b50b4e8ebec55c3830af94a5957ac4 Mon Sep 17 00:00:00 2001 From: rohansun Date: Thu, 5 Sep 2024 10:09:04 -0400 Subject: [PATCH 3/3] Fix linting issues by running Black --- metadata-ingestion/src/datahub/ingestion/source/sql_queries.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py index b38c4080397f08..f3e8e774e4388f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql_queries.py @@ -72,7 +72,6 @@ class SqlQueriesSourceConfig(PlatformInstanceConfigMixin, EnvConfigMixin): ) - class SqlQueriesSourceReport(SourceReport): num_queries_parsed: int = 0 num_table_parse_failures: int = 0 @@ -182,7 +181,7 @@ def _process_query(self, entry: "QueryEntry") -> Iterable[MetadataWorkUnit]: schema_resolver=self.schema_resolver, default_db=self.config.default_db, default_schema=self.config.default_schema, - default_dialect=self.config.default_dialect, + default_dialect=self.config.default_dialect, ) if result.debug_info.table_error: logger.info(f"Error parsing table lineage, {result.debug_info.table_error}")