fix(snowflake): user_email_pattern

datahub-project · Jan 31, 2025 · 36a752b · 36a752b
1 parent bde5c73
commit 36a752b
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py b/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py
@@ -273,10 +273,11 @@ def aggregate_event(
                 resource=resource,
             ),
         ).add_read_entry(
-            user,
-            query,
-            fields,
+            user_email=user,
+            query=query,
+            fields=fields,
             count=count,
+            user_email_pattern=self.config.user_email_pattern,
         )
 
     def generate_workunits(

diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
@@ -584,6 +584,15 @@ def add(
             TableSwap,
         ],
     ) -> None:
+        if (
+            isinstance(item, PreparsedQuery | ObservedQuery)
+            and item.user
+            and self.usage_config
+            and not self.usage_config.user_email_pattern.allowed(item.user.urn())
+        ):
+            logger.info(f"Skipping query from user: {item.user}")
+            return
+
         if isinstance(item, KnownQueryLineageInfo):
             self.add_known_query_lineage(item)
         elif isinstance(item, KnownLineageMapping):