diff --git a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md index 4cfbc470e8c239..420715e5312cb6 100644 --- a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md +++ b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md @@ -15,6 +15,8 @@ grant operate, usage on warehouse "" to role datahub_role; grant usage on DATABASE "" to role datahub_role; grant usage on all schemas in database "" to role datahub_role; grant usage on future schemas in database "" to role datahub_role; +grant select on all streams in database "> to role datahub_role; +grant select on future streams in database "> to role datahub_role; // If you are NOT using Snowflake Profiling or Classification feature: Grant references privileges to your tables and views grant references on all tables in database "" to role datahub_role; @@ -50,9 +52,12 @@ The details of each granted privilege can be viewed in [snowflake docs](https:// If the warehouse is already running during ingestion or has auto-resume enabled, this permission is not required. - `usage` is required for us to run queries using the warehouse -- `usage` on `database` and `schema` are required because without it tables and views inside them are not accessible. If an admin does the required grants on `table` but misses the grants on `schema` or the `database` in which the table/view exists then we will not be able to get metadata for the table/view. +- `usage` on `database` and `schema` are required because without it tables, views, and streams inside them are not accessible. If an admin does the required grants on `table` but misses the grants on `schema` or the `database` in which the table/view/stream exists then we will not be able to get metadata for the table/view/stream. - If metadata is required only on some schemas then you can grant the usage privilieges only on a particular schema like - +```sql +grant usage on schema ""."" to role datahub_role; +``` +- `select` on `streams` is required in order for stream definitions to be available. This does not allow selecting of the data (not required) unless the underlying dataset has select access as well. ```sql grant usage on schema ""."" to role datahub_role; ``` diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index a5eecf198a9b49..8eb53259df8062 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -24,6 +24,7 @@ class DatasetSubTypes(StrEnum): SAC_LIVE_DATA_MODEL = "Live Data Model" NEO4J_NODE = "Neo4j Node" NEO4J_RELATIONSHIP = "Neo4j Relationship" + SNOWFLAKE_STREAM = "Snowflake Stream" # TODO: Create separate entity... NOTEBOOK = "Notebook" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py index fc464cc5ea9bb5..84ccd2c67c95a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py @@ -53,6 +53,7 @@ class SnowflakeObjectDomain(StrEnum): SCHEMA = "schema" COLUMN = "column" ICEBERG_TABLE = "iceberg table" + STREAM = "stream" GENERIC_PERMISSION_ERROR_KEY = "permission-error" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 5f732e2621656f..22492068fc7791 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -98,6 +98,11 @@ class SnowflakeFilterConfig(SQLFilterConfig): ) # table_pattern and view_pattern are inherited from SQLFilterConfig + stream_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for streams to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", + ) + match_fully_qualified_names: bool = Field( default=False, description="Whether `schema_pattern` is matched against fully qualified schema name `.`.", @@ -268,6 +273,11 @@ class SnowflakeV2Config( description="List of regex patterns for tags to include in ingestion. Only used if `extract_tags` is enabled.", ) + include_streams: bool = Field( + default=True, + description="If enabled, streams will be ingested as separate entities from tables/views.", + ) + structured_property_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), description=( diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index 69d0b62a8edfdf..eb015f9d13281f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -49,6 +49,7 @@ from datahub.sql_parsing.schema_resolver import SchemaResolver from datahub.sql_parsing.sql_parsing_aggregator import ( KnownLineageMapping, + ObservedQuery, PreparsedQuery, SqlAggregatorReport, SqlParsingAggregator, @@ -241,7 +242,13 @@ def get_workunits_internal( use_cached_audit_log = audit_log_file.exists() queries: FileBackedList[ - Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] + Union[ + KnownLineageMapping, + PreparsedQuery, + TableRename, + TableSwap, + ObservedQuery, + ] ] if use_cached_audit_log: logger.info("Using cached audit log") @@ -252,7 +259,13 @@ def get_workunits_internal( shared_connection = ConnectionWrapper(audit_log_file) queries = FileBackedList(shared_connection) - entry: Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] + entry: Union[ + KnownLineageMapping, + PreparsedQuery, + TableRename, + TableSwap, + ObservedQuery, + ] with self.report.copy_history_fetch_timer: for entry in self.fetch_copy_history(): @@ -329,7 +342,7 @@ def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: def fetch_query_log( self, users: UsersMapping - ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap]]: + ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap, ObservedQuery]]: query_log_query = _build_enriched_query_log_query( start_time=self.config.window.start_time, end_time=self.config.window.end_time, @@ -362,7 +375,7 @@ def fetch_query_log( def _parse_audit_log_row( self, row: Dict[str, Any], users: UsersMapping - ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery]]: + ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery, ObservedQuery]]: json_fields = { "DIRECT_OBJECTS_ACCESSED", "OBJECTS_MODIFIED", @@ -398,6 +411,34 @@ def _parse_audit_log_row( pass else: return None + + user = CorpUserUrn( + self.identifiers.get_user_identifier( + res["user_name"], users.get(res["user_name"]) + ) + ) + + # Use direct_objects_accessed instead objects_modified + # objects_modified returns $SYS_VIEW_X with no mapping + has_stream_objects = any( + obj.get("objectDomain") == "Stream" for obj in direct_objects_accessed + ) + + # If a stream is used, default to query parsing. + if has_stream_objects: + logger.debug("Found matching stream object") + return ObservedQuery( + query=res["query_text"], + session_id=res["session_id"], + timestamp=res["query_start_time"].astimezone(timezone.utc), + user=user, + default_db=res["default_db"], + default_schema=res["default_schema"], + query_hash=get_query_fingerprint( + res["query_text"], self.identifiers.platform, fast=True + ), + ) + upstreams = [] column_usage = {} @@ -460,12 +501,6 @@ def _parse_audit_log_row( ) ) - user = CorpUserUrn( - self.identifiers.get_user_identifier( - res["user_name"], users.get(res["user_name"]) - ) - ) - timestamp: datetime = res["query_start_time"] timestamp = timestamp.astimezone(timezone.utc) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 40bcfb514efd23..b68b01dbb0243c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -9,6 +9,7 @@ from datahub.utilities.prefix_batch_builder import PrefixGroup SHOW_VIEWS_MAX_PAGE_SIZE = 10000 +SHOW_STREAM_MAX_PAGE_SIZE = 10000 def create_deny_regex_sql_filter( @@ -36,6 +37,7 @@ class SnowflakeQuery: SnowflakeObjectDomain.VIEW.capitalize(), SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize(), SnowflakeObjectDomain.ICEBERG_TABLE.capitalize(), + SnowflakeObjectDomain.STREAM.capitalize(), } ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = "({})".format( @@ -44,7 +46,8 @@ class SnowflakeQuery: ACCESS_HISTORY_TABLE_DOMAINS_FILTER = ( "(" f"'{SnowflakeObjectDomain.TABLE.capitalize()}'," - f"'{SnowflakeObjectDomain.VIEW.capitalize()}'" + f"'{SnowflakeObjectDomain.VIEW.capitalize()}'," + f"'{SnowflakeObjectDomain.STREAM.capitalize()}'," ")" ) @@ -952,3 +955,19 @@ def dmf_assertion_results(start_time_millis: int, end_time_millis: int) -> str: @staticmethod def get_all_users() -> str: return """SELECT name as "NAME", email as "EMAIL" FROM SNOWFLAKE.ACCOUNT_USAGE.USERS""" + + @staticmethod + def streams_for_database( + db_name: str, + limit: int = SHOW_STREAM_MAX_PAGE_SIZE, + stream_pagination_marker: Optional[str] = None, + ) -> str: + # SHOW STREAMS can return a maximum of 10000 rows. + # https://docs.snowflake.com/en/sql-reference/sql/show-streams#usage-notes + assert limit <= SHOW_STREAM_MAX_PAGE_SIZE + + # To work around this, we paginate through the results using the FROM clause. + from_clause = ( + f"""FROM '{stream_pagination_marker}'""" if stream_pagination_marker else "" + ) + return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index b24471f8666afa..f658642ab9e57f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -103,6 +103,7 @@ class SnowflakeV2Report( schemas_scanned: int = 0 databases_scanned: int = 0 tags_scanned: int = 0 + streams_scanned: int = 0 include_usage_stats: bool = False include_operational_stats: bool = False @@ -112,6 +113,7 @@ class SnowflakeV2Report( table_lineage_query_secs: float = -1 external_lineage_queries_secs: float = -1 num_tables_with_known_upstreams: int = 0 + num_streams_with_known_upstreams: int = 0 num_upstream_lineage_edge_parsing_failed: int = 0 num_secure_views_missing_definition: int = 0 @@ -129,6 +131,8 @@ class SnowflakeV2Report( num_get_tags_for_object_queries: int = 0 num_get_tags_on_columns_for_table_queries: int = 0 + num_get_streams_for_schema_queries: int = 0 + rows_zero_objects_modified: int = 0 _processed_tags: MutableSet[str] = field(default_factory=set) @@ -155,6 +159,8 @@ def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: return self._scanned_tags.add(name) self.tags_scanned += 1 + elif ent_type == "stream": + self.streams_scanned += 1 else: raise KeyError(f"Unknown entity {ent_type}.") diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 173024aec0cf38..9fd497bebb7bb9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -14,7 +14,7 @@ ) from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView from datahub.utilities.file_backed_collections import FileBackedDict -from datahub.utilities.prefix_batch_builder import build_prefix_batches +from datahub.utilities.prefix_batch_builder import PrefixGroup, build_prefix_batches from datahub.utilities.serialized_lru_cache import serialized_lru_cache logger: logging.Logger = logging.getLogger(__name__) @@ -118,6 +118,7 @@ class SnowflakeSchema: comment: Optional[str] tables: List[str] = field(default_factory=list) views: List[str] = field(default_factory=list) + streams: List[str] = field(default_factory=list) tags: Optional[List[SnowflakeTag]] = None @@ -131,6 +132,29 @@ class SnowflakeDatabase: tags: Optional[List[SnowflakeTag]] = None +@dataclass +class SnowflakeStream: + name: str + created: datetime + owner: str + source_type: str + type: str + stale: str + mode: str + invalid_reason: str + owner_role_type: str + database_name: str + schema_name: str + table_name: str + comment: Optional[str] + columns: List[SnowflakeColumn] = field(default_factory=list) + stale_after: Optional[datetime] = None + base_tables: Optional[str] = None + tags: Optional[List[SnowflakeTag]] = None + column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict) + last_altered: Optional[datetime] = None + + class _SnowflakeTagCache: def __init__(self) -> None: # self._database_tags[] = list of tags applied to database @@ -208,6 +232,7 @@ def as_obj(self) -> Dict[str, Dict[str, int]]: self.get_tables_for_database, self.get_views_for_database, self.get_columns_for_schema, + self.get_streams_for_database, self.get_pk_constraints_for_schema, self.get_fk_constraints_for_schema, ] @@ -414,9 +439,18 @@ def get_columns_for_schema( # For massive schemas, use a FileBackedDict to avoid memory issues. columns = FileBackedDict() - object_batches = build_prefix_batches( - all_objects, max_batch_size=10000, max_groups_in_batch=5 - ) + # Single prefix table case (for streams) + if len(all_objects) == 1: + object_batches = [ + [PrefixGroup(prefix=all_objects[0], names=[], exact_match=True)] + ] + else: + # Build batches for full schema scan + object_batches = build_prefix_batches( + all_objects, max_batch_size=10000, max_groups_in_batch=5 + ) + + # Process batches for batch_index, object_batch in enumerate(object_batches): if batch_index > 0: logger.info( @@ -594,3 +628,63 @@ def get_tags_on_columns_for_table( tags[column_name].append(snowflake_tag) return tags + + @serialized_lru_cache(maxsize=1) + def get_streams_for_database( + self, db_name: str + ) -> Dict[str, List[SnowflakeStream]]: + page_limit = SHOW_VIEWS_MAX_PAGE_SIZE + + streams: Dict[str, List[SnowflakeStream]] = {} + + first_iteration = True + stream_pagination_marker: Optional[str] = None + while first_iteration or stream_pagination_marker is not None: + cur = self.connection.query( + SnowflakeQuery.streams_for_database( + db_name, + limit=page_limit, + stream_pagination_marker=stream_pagination_marker, + ) + ) + + first_iteration = False + stream_pagination_marker = None + + result_set_size = 0 + for stream in cur: + result_set_size += 1 + + stream_name = stream["name"] + schema_name = stream["schema_name"] + if schema_name not in streams: + streams[schema_name] = [] + streams[stream["schema_name"]].append( + SnowflakeStream( + name=stream["name"], + created=stream["created_on"], + owner=stream["owner"], + comment=stream["comment"], + source_type=stream["source_type"], + type=stream["type"], + stale=stream["stale"], + mode=stream["mode"], + database_name=stream["database_name"], + schema_name=stream["schema_name"], + invalid_reason=stream["invalid_reason"], + owner_role_type=stream["owner_role_type"], + stale_after=stream["stale_after"], + table_name=stream["table_name"], + base_tables=stream["base_tables"], + last_altered=stream["created_on"], + ) + ) + + if result_set_size >= page_limit: + # If we hit the limit, we need to send another request to get the next page. + logger.info( + f"Fetching next page of streams for {db_name} - after {stream_name}" + ) + stream_pagination_marker = stream_name + + return streams diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 04bc51f1ebd3f5..e945ab53b67657 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -48,6 +48,7 @@ SnowflakeFK, SnowflakePK, SnowflakeSchema, + SnowflakeStream, SnowflakeTable, SnowflakeTag, SnowflakeView, @@ -58,6 +59,7 @@ SnowflakeIdentifierBuilder, SnowflakeStructuredReportMixin, SnowsightUrlBuilder, + split_qualified_name, ) from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, @@ -70,6 +72,7 @@ ) from datahub.ingestion.source_report.ingestion_stage import ( EXTERNAL_TABLE_DDL_LINEAGE, + LINEAGE_EXTRACTION, METADATA_EXTRACTION, PROFILING, ) @@ -82,6 +85,7 @@ TimeStamp, ) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageTypeClass, DatasetProperties, ViewProperties, ) @@ -418,73 +422,120 @@ def _process_schema( schema_name = snowflake_schema.name if self.config.extract_tags != TagOption.skip: - snowflake_schema.tags = self.tag_extractor.get_tags_on_object( - schema_name=schema_name, db_name=db_name, domain="schema" - ) + self._process_tags(snowflake_schema, schema_name, db_name, domain="schema") if self.config.include_technical_schema: yield from self.gen_schema_containers(snowflake_schema, db_name) - # We need to do this first so that we can use it when fetching columns. + tables, views, streams = [], [], [] + if self.config.include_tables: tables = self.fetch_tables_for_schema( snowflake_schema, db_name, schema_name ) + db_tables[schema_name] = tables + yield from self._process_tables( + tables, snowflake_schema, db_name, schema_name + ) + if self.config.include_views: views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) + yield from self._process_views( + views, snowflake_schema, db_name, schema_name + ) - if self.config.include_tables: - db_tables[schema_name] = tables + if self.config.include_streams: + self.report.num_get_streams_for_schema_queries += 1 + streams = self.fetch_streams_for_schema( + snowflake_schema, db_name, schema_name + ) + yield from self._process_streams(streams, snowflake_schema, db_name) - if self.config.include_technical_schema: - data_reader = self.make_data_reader() - for table in tables: - table_wu_generator = self._process_table( - table, snowflake_schema, db_name - ) + if self.config.include_technical_schema and snowflake_schema.tags: + yield from self._process_tags_in_schema(snowflake_schema) - yield from classification_workunit_processor( - table_wu_generator, - self.classification_handler, - data_reader, - [db_name, schema_name, table.name], - ) + if ( + not snowflake_schema.views + and not snowflake_schema.tables + and not snowflake_schema.streams + ): + self.structured_reporter.info( + title="No tables/views/streams found in schema", + message="If objects exist, please grant REFERENCES or SELECT permissions on them.", + context=f"{db_name}.{schema_name}", + ) - if self.config.include_views: - if self.aggregator: - for view in views: - view_identifier = self.identifiers.get_dataset_identifier( + def _process_tags(self, snowflake_schema, schema_name, db_name, domain): + snowflake_schema.tags = self.tag_extractor.get_tags_on_object( + schema_name=schema_name, db_name=db_name, domain=domain + ) + + def _process_tables( + self, + tables: List[SnowflakeTable], + snowflake_schema: SnowflakeSchema, + db_name: str, + schema_name: str, + ) -> Iterable[MetadataWorkUnit]: + if self.config.include_technical_schema: + data_reader = self.make_data_reader() + for table in tables: + table_wu_generator = self._process_table( + table, snowflake_schema, db_name + ) + yield from classification_workunit_processor( + table_wu_generator, + self.classification_handler, + data_reader, + [db_name, schema_name, table.name], + ) + + def _process_views( + self, + views: List[SnowflakeView], + snowflake_schema: SnowflakeSchema, + db_name: str, + schema_name: str, + ) -> Iterable[MetadataWorkUnit]: + if self.aggregator: + for view in views: + view_identifier = self.identifiers.get_dataset_identifier( + view.name, schema_name, db_name + ) + if view.is_secure and not view.view_definition: + view.view_definition = self.fetch_secure_view_definition( view.name, schema_name, db_name ) - if view.is_secure and not view.view_definition: - view.view_definition = self.fetch_secure_view_definition( - view.name, schema_name, db_name - ) - if view.view_definition: - self.aggregator.add_view_definition( - view_urn=self.identifiers.gen_dataset_urn(view_identifier), - view_definition=view.view_definition, - default_db=db_name, - default_schema=schema_name, - ) - elif view.is_secure: - self.report.num_secure_views_missing_definition += 1 + if view.view_definition: + self.aggregator.add_view_definition( + view_urn=self.identifiers.gen_dataset_urn(view_identifier), + view_definition=view.view_definition, + default_db=db_name, + default_schema=schema_name, + ) + elif view.is_secure: + self.report.num_secure_views_missing_definition += 1 - if self.config.include_technical_schema: - for view in views: - yield from self._process_view(view, snowflake_schema, db_name) + if self.config.include_technical_schema: + for view in views: + yield from self._process_view(view, snowflake_schema, db_name) - if self.config.include_technical_schema and snowflake_schema.tags: + def _process_streams( + self, + streams: List[SnowflakeStream], + snowflake_schema: SnowflakeSchema, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + for stream in streams: + yield from self._process_stream(stream, snowflake_schema, db_name) + + def _process_tags_in_schema( + self, snowflake_schema: SnowflakeSchema + ) -> Iterable[MetadataWorkUnit]: + if snowflake_schema.tags: for tag in snowflake_schema.tags: yield from self._process_tag(tag) - if not snowflake_schema.views and not snowflake_schema.tables: - self.structured_reporter.info( - title="No tables/views found in schema", - message="If tables exist, please grant REFERENCES or SELECT permissions on them.", - context=f"{db_name}.{schema_name}", - ) - def fetch_secure_view_definition( self, table_name: str, schema_name: str, db_name: str ) -> Optional[str]: @@ -725,7 +776,7 @@ def _format_tags_as_structured_properties( def gen_dataset_workunits( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> Iterable[MetadataWorkUnit]: @@ -783,7 +834,9 @@ def gen_dataset_workunits( subTypes = SubTypes( typeNames=( - [DatasetSubTypes.VIEW] + [DatasetSubTypes.SNOWFLAKE_STREAM] + if isinstance(table, SnowflakeStream) + else [DatasetSubTypes.VIEW] if isinstance(table, SnowflakeView) else [DatasetSubTypes.TABLE] ) @@ -838,28 +891,50 @@ def gen_dataset_workunits( def get_dataset_properties( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> DatasetProperties: custom_properties = {} if isinstance(table, SnowflakeTable): - if table.clustering_key: - custom_properties["CLUSTERING_KEY"] = table.clustering_key - - if table.is_hybrid: - custom_properties["IS_HYBRID"] = "true" - - if table.is_dynamic: - custom_properties["IS_DYNAMIC"] = "true" - - if table.is_iceberg: - custom_properties["IS_ICEBERG"] = "true" + custom_properties.update( + { + k: v + for k, v in { + "CLUSTERING_KEY": table.clustering_key, + "IS_HYBRID": "true" if table.is_hybrid else None, + "IS_DYNAMIC": "true" if table.is_dynamic else None, + "IS_ICEBERG": "true" if table.is_iceberg else None, + }.items() + if v + } + ) if isinstance(table, SnowflakeView) and table.is_secure: custom_properties["IS_SECURE"] = "true" + elif isinstance(table, SnowflakeStream): + custom_properties.update( + { + k: v + for k, v in { + "SOURCE_TYPE": table.source_type, + "TYPE": table.type, + "STALE": table.stale, + "MODE": table.mode, + "INVALID_REASON": table.invalid_reason, + "OWNER_ROLE_TYPE": table.owner_role_type, + "TABLE_NAME": table.table_name, + "BASE_TABLES": table.base_tables, + "STALE_AFTER": table.stale_after.isoformat() + if table.stale_after + else None, + }.items() + if v + } + ) + return DatasetProperties( name=table.name, created=( @@ -927,7 +1002,9 @@ def gen_tag_as_structured_property_workunits( ).as_workunit() def gen_column_tags_as_structured_properties( - self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView] + self, + dataset_urn: str, + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], ) -> Iterable[MetadataWorkUnit]: for column_name in table.column_tags: schema_field_urn = SchemaFieldUrn(dataset_urn, column_name).urn() @@ -940,7 +1017,7 @@ def gen_column_tags_as_structured_properties( def gen_schema_metadata( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> SchemaMetadata: @@ -1232,3 +1309,158 @@ def _external_tables_ddl_lineage( "External table ddl lineage extraction failed", exc=e, ) + + def fetch_streams_for_schema( + self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str + ) -> List[SnowflakeStream]: + try: + streams: List[SnowflakeStream] = [] + for stream in self.get_streams_for_schema(schema_name, db_name): + stream_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + + self.report.report_entity_scanned(stream_identifier, "stream") + + if not self.filters.is_dataset_pattern_allowed( + stream_identifier, SnowflakeObjectDomain.STREAM + ): + self.report.report_dropped(stream_identifier) + else: + streams.append(stream) + snowflake_schema.streams = [stream.name for stream in streams] + return streams + except Exception as e: + if isinstance(e, SnowflakePermissionError): + error_msg = f"Failed to get streams for schema {db_name}.{schema_name}. Please check permissions." + raise SnowflakePermissionError(error_msg) from e.__cause__ + else: + self.structured_reporter.warning( + "Failed to get streams for schema", + f"{db_name}.{schema_name}", + exc=e, + ) + return [] + + def get_streams_for_schema( + self, schema_name: str, db_name: str + ) -> List[SnowflakeStream]: + streams = self.data_dictionary.get_streams_for_database(db_name) + + return streams.get(schema_name, []) + + def _process_stream( + self, + stream: SnowflakeStream, + snowflake_schema: SnowflakeSchema, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + schema_name = snowflake_schema.name + + try: + # Retrieve and register the schema without metadata to prevent columns from mapping upstream + stream.columns = self.get_columns_for_stream(stream.table_name) + yield from self.gen_dataset_workunits(stream, schema_name, db_name) + + if self.config.include_column_lineage: + with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): + self.populate_stream_upstreams(stream, db_name, schema_name) + + except Exception as e: + self.structured_reporter.warning( + "Failed to get columns for stream:", stream.name, exc=e + ) + + def get_columns_for_stream( + self, + source_object: str, # Qualified name of source table/view + ) -> List[SnowflakeColumn]: + """ + Get column information for a stream by getting source object columns and adding metadata columns. + Stream includes all columns from source object plus metadata columns like: + - METADATA$ACTION + - METADATA$ISUPDATE + - METADATA$ROW_ID + """ + columns: List[SnowflakeColumn] = [] + + source_parts = split_qualified_name(source_object) + + source_db, source_schema, source_name = source_parts + + # Get columns from source object + source_columns = self.data_dictionary.get_columns_for_schema( + source_schema, source_db, itertools.chain([source_name]) + ).get(source_name, []) + + # Add all source columns + columns.extend(source_columns) + + # Add standard stream metadata columns + metadata_columns = [ + SnowflakeColumn( + name="METADATA$ACTION", + ordinal_position=len(columns) + 1, + is_nullable=False, + data_type="VARCHAR", + comment="Type of DML operation (INSERT/DELETE)", + character_maximum_length=10, + numeric_precision=None, + numeric_scale=None, + ), + SnowflakeColumn( + name="METADATA$ISUPDATE", + ordinal_position=len(columns) + 2, + is_nullable=False, + data_type="BOOLEAN", + comment="Whether row is from UPDATE operation", + character_maximum_length=None, + numeric_precision=None, + numeric_scale=None, + ), + SnowflakeColumn( + name="METADATA$ROW_ID", + ordinal_position=len(columns) + 3, + is_nullable=False, + data_type="NUMBER", + comment="Unique row identifier", + character_maximum_length=None, + numeric_precision=38, + numeric_scale=0, + ), + ] + + columns.extend(metadata_columns) + + return columns + + def populate_stream_upstreams( + self, stream: SnowflakeStream, db_name: str, schema_name: str + ) -> None: + """ + Populate Streams upstream tables + """ + self.report.num_streams_with_known_upstreams += 1 + if self.aggregator: + source_parts = split_qualified_name(stream.table_name) + source_db, source_schema, source_name = source_parts + + dataset_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + dataset_urn = self.identifiers.gen_dataset_urn(dataset_identifier) + + upstream_identifier = self.identifiers.get_dataset_identifier( + source_name, source_schema, source_db + ) + upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) + + logger.debug( + f"""upstream_urn: {upstream_urn}, downstream_urn: {dataset_urn}""" + ) + + self.aggregator.add_known_lineage_mapping( + upstream_urn=upstream_urn, + downstream_urn=dataset_urn, + lineage_type=DatasetLineageTypeClass.COPY, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 030edfde4ca1da..1e95597061577e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -124,19 +124,20 @@ def is_dataset_pattern_allowed( SnowflakeObjectDomain.VIEW, SnowflakeObjectDomain.MATERIALIZED_VIEW, SnowflakeObjectDomain.ICEBERG_TABLE, + SnowflakeObjectDomain.STREAM, ): return False if _is_sys_table(dataset_name): return False - dataset_params = _split_qualified_name(dataset_name) + dataset_params = split_qualified_name(dataset_name) if len(dataset_params) != 3: self.structured_reporter.info( title="Unexpected dataset pattern", message=f"Found a {dataset_type} with an unexpected number of parts. Database and schema filtering will not work as expected, but table filtering will still work.", context=dataset_name, ) - # We fall-through here so table/view filtering still works. + # We fall-through here so table/view/stream filtering still works. if ( len(dataset_params) >= 1 @@ -169,6 +170,14 @@ def is_dataset_pattern_allowed( ): return False + if ( + dataset_type.lower() == SnowflakeObjectDomain.STREAM + and not self.filter_config.stream_pattern.allowed( + _cleanup_qualified_name(dataset_name, self.structured_reporter) + ) + ): + return False + return True @@ -183,17 +192,17 @@ def _is_sys_table(table_name: str) -> bool: return table_name.lower().startswith("sys$") -def _split_qualified_name(qualified_name: str) -> List[str]: +def split_qualified_name(qualified_name: str) -> List[str]: """ Split a qualified name into its constituent parts. - >>> _split_qualified_name("db.my_schema.my_table") + >>> split_qualified_name("db.my_schema.my_table") ['db', 'my_schema', 'my_table'] - >>> _split_qualified_name('"db"."my_schema"."my_table"') + >>> split_qualified_name('"db"."my_schema"."my_table"') ['db', 'my_schema', 'my_table'] - >>> _split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"') + >>> split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"') ['TEST_DB', 'TEST_SCHEMA', 'TABLE.WITH.DOTS'] - >>> _split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE') + >>> split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE') ['TEST_DB', 'SCHEMA.WITH.DOTS', 'MY_TABLE'] """ @@ -231,7 +240,7 @@ def _split_qualified_name(qualified_name: str) -> List[str]: def _cleanup_qualified_name( qualified_name: str, structured_reporter: SourceReport ) -> str: - name_parts = _split_qualified_name(qualified_name) + name_parts = split_qualified_name(qualified_name) if len(name_parts) != 3: if not _is_sys_table(qualified_name): structured_reporter.info( diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index b4ef2180d71d45..5a00978c5e0d8c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -505,15 +505,25 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: for schema in db.schemas for table_name in schema.views ] + discovered_streams: List[str] = [ + self.identifiers.get_dataset_identifier(stream_name, schema.name, db.name) + for db in databases + for schema in db.schemas + for stream_name in schema.streams + ] - if len(discovered_tables) == 0 and len(discovered_views) == 0: + if ( + len(discovered_tables) == 0 + and len(discovered_views) == 0 + and len(discovered_streams) == 0 + ): self.structured_reporter.failure( GENERIC_PERMISSION_ERROR_KEY, - "No tables/views found. Please check permissions.", + "No tables/views/streams found. Please check permissions.", ) return - discovered_datasets = discovered_tables + discovered_views + discovered_datasets = discovered_tables + discovered_views + discovered_streams if self.config.use_queries_v2: with self.report.new_stage(f"*: {VIEW_PARSING}"): diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 7b4f5abe1cd462..d059864467ead2 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -7,9 +7,11 @@ from datahub.configuration.time_window_config import BucketDuration from datahub.ingestion.source.snowflake import snowflake_query from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.utilities.prefix_batch_builder import PrefixGroup NUM_TABLES = 10 NUM_VIEWS = 2 +NUM_STREAMS = 1 NUM_COLS = 10 NUM_OPS = 10 NUM_USAGE = 0 @@ -175,6 +177,7 @@ def default_query_results( # noqa: C901 query, num_tables=NUM_TABLES, num_views=NUM_VIEWS, + num_streams=NUM_STREAMS, num_cols=NUM_COLS, num_ops=NUM_OPS, num_usages=NUM_USAGE, @@ -272,6 +275,27 @@ def default_query_results( # noqa: C901 for view_idx in range(1, num_views + 1) if is_secure(view_idx) ] + elif query == SnowflakeQuery.columns_for_schema( + "TEST_SCHEMA", + "TEST_DB", + [PrefixGroup(prefix="TABLE_1", names=[], exact_match=True)], + ): + return [ + { + "TABLE_CATALOG": "TEST_DB", + "TABLE_SCHEMA": "TEST_SCHEMA", + "TABLE_NAME": "TABLE_1", + "COLUMN_NAME": f"COL_{col_idx}", + "ORDINAL_POSITION": col_idx, + "IS_NULLABLE": "NO", + "DATA_TYPE": "TEXT" if col_idx > 1 else "NUMBER", + "COMMENT": "Comment for column", + "CHARACTER_MAXIMUM_LENGTH": 255 if col_idx > 1 else None, + "NUMERIC_PRECISION": None if col_idx > 1 else 38, + "NUMERIC_SCALE": None if col_idx > 1 else 0, + } + for col_idx in range(1, num_cols + 1) + ] elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"): return [ { @@ -293,6 +317,28 @@ def default_query_results( # noqa: C901 ) for col_idx in range(1, num_cols + 1) ] + elif query == SnowflakeQuery.streams_for_database("TEST_DB"): + # TODO: Add tests for stream pagination. + return [ + { + "created_on": datetime(2021, 6, 8, 0, 0, 0, 0, tzinfo=timezone.utc), + "name": f"STREAM_{stream_idx}", + "database_name": "TEST_DB", + "schema_name": "TEST_SCHEMA", + "owner": "ACCOUNTADMIN", + "comment": f"Comment for Stream {stream_idx}", + "table_name": f"TEST_DB.TEST_SCHEMA.TABLE_{stream_idx}", + "source_type": "Table", + "base_tables": f"TEST_DB.TEST_SCHEMA.TABLE_{stream_idx}", + "type": "DELTA", + "stale": "false", + "mode": "DEFAULT", + "stale_after": datetime(2021, 6, 22, 0, 0, 0, 0, tzinfo=timezone.utc), + "invalid_reason": None, + "owner_role_type": "ROLE", + } + for stream_idx in range(1, num_streams + 1) + ] elif query in ( SnowflakeQuery.use_database("TEST_DB"), SnowflakeQuery.show_primary_keys_for_schema("TEST_SCHEMA", "TEST_DB"), diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 48ec46af069cef..4b3bd16deceeae 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -16,16 +16,16 @@ "description": "Comment for TEST_DB", "env": "PROD", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_08_20-16_18_24", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -40,8 +40,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -56,8 +56,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -74,8 +74,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -94,8 +94,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -110,8 +110,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -127,8 +127,24 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -150,16 +166,16 @@ "description": "comment for TEST_DB.TEST_SCHEMA", "env": "PROD", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_08_20-16_18_24", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -174,8 +190,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -190,8 +206,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -208,8 +224,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -228,24 +244,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -265,8 +265,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -281,8 +281,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -329,7 +329,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -353,7 +353,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -377,7 +377,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -478,8 +478,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -498,17 +498,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -523,8 +523,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -541,8 +541,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -566,8 +566,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -582,8 +582,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -630,7 +630,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -654,7 +654,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -678,7 +678,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -779,8 +779,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -799,17 +799,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -824,8 +824,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -842,8 +842,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -867,8 +867,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -883,8 +883,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -931,7 +931,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -955,7 +955,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -979,7 +979,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1080,8 +1080,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1100,17 +1100,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1125,8 +1125,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1143,8 +1143,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1168,8 +1168,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1184,8 +1184,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1232,7 +1232,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1256,7 +1256,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1280,7 +1280,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1381,8 +1381,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1401,17 +1401,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1426,8 +1426,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1444,8 +1444,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1469,8 +1469,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1485,8 +1485,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1533,7 +1533,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1557,7 +1557,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1581,7 +1581,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1682,8 +1682,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1702,17 +1702,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1727,8 +1727,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1745,8 +1745,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1770,8 +1770,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1786,8 +1786,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1834,7 +1834,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1858,7 +1858,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1882,7 +1882,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1983,8 +1983,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2003,17 +2003,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2028,8 +2028,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2046,8 +2046,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2071,8 +2071,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2087,8 +2087,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2135,7 +2135,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2159,7 +2159,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2183,7 +2183,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2284,8 +2284,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2304,17 +2304,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2329,8 +2329,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2347,8 +2347,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2372,8 +2372,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2388,8 +2388,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2436,7 +2436,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2460,7 +2460,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2484,7 +2484,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2585,8 +2585,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2605,17 +2605,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2630,8 +2630,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2648,8 +2648,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2673,8 +2673,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2689,8 +2689,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2737,7 +2737,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2761,7 +2761,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2785,7 +2785,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2886,8 +2886,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2906,17 +2906,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2931,8 +2931,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2949,8 +2949,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2974,8 +2974,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2990,8 +2990,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3038,7 +3038,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3062,7 +3062,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3086,7 +3086,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3187,8 +3187,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3207,17 +3207,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3232,8 +3232,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3250,8 +3250,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3275,8 +3275,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3292,8 +3292,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3308,8 +3308,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3479,8 +3479,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3499,17 +3499,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", "description": "Comment for View", "created": { - "time": 1623090600000 + "time": 1623124800000 }, "lastModified": { - "time": 1623090600000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_12_16-15_30_20-649nax", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3524,8 +3524,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3542,8 +3542,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3561,7 +3561,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3585,8 +3585,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3602,8 +3602,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3619,8 +3619,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3636,8 +3636,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3652,8 +3652,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3816,8 +3816,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3834,17 +3834,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", "description": "Comment for View", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3859,8 +3859,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3877,8 +3877,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3903,33 +3903,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3947,179 +3922,646 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "browsePathsV2", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "status", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "removed": false } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "schemaMetadata", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00+00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623110400000 + }, + "lastModified": { + "time": 1623110400000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00+00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623110400000 }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "lastModified": { + "time": 1623110400000 + }, + "tags": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516740, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4130,19 +4572,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516750, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4153,19 +4595,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516750, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4176,194 +4618,175 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "datasetProfile", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 1615443388097, - "actor": "urn:li:corpuser:_ingestion" - }, - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "type": "VIEW", - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - } - ] + "timestampMillis": 1738040516751, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1738040516751, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetProfile", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_1\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" + "timestampMillis": 1738040516752, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1738040516752, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1738040516753, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1738040516754, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1738040516754, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -4378,216 +4801,160 @@ "time": 0, "actor": "urn:li:corpuser:_ingestion" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "VIEW", - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "type": "COPY" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$action)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$action)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$isupdate)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$isupdate)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$row_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$row_id)" + ], + "confidenceScore": 1.0 } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4766,13 +5133,116 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_1\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -4789,7 +5259,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ], "fineGrainedLineages": [ @@ -4800,10 +5270,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4812,10 +5282,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4824,10 +5294,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4836,10 +5306,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4848,10 +5318,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4860,10 +5330,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4872,10 +5342,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4884,10 +5354,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4896,10 +5366,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -4908,33 +5378,17 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4962,41 +5416,82 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_2\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "platform": "urn:li:dataPlatform:snowflake" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5013,7 +5508,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ], "fineGrainedLineages": [ @@ -5024,10 +5519,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5036,10 +5531,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5048,10 +5543,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5060,10 +5555,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5072,10 +5567,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5084,10 +5579,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5096,10 +5591,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5108,10 +5603,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5120,10 +5615,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5132,17 +5627,45 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_2\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5154,9 +5677,6 @@ "aspect": { "json": { "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, @@ -5195,7 +5715,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5211,66 +5731,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5287,7 +5754,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" } ], "fineGrainedLineages": [ @@ -5298,10 +5765,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5310,10 +5777,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5322,10 +5789,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5334,10 +5801,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5346,10 +5813,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5358,10 +5825,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5370,10 +5837,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5382,10 +5849,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5394,10 +5861,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5406,23 +5873,120 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_3\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5439,7 +6003,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ], "fineGrainedLineages": [ @@ -5450,10 +6014,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5462,10 +6026,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5474,10 +6038,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5486,10 +6050,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5498,10 +6062,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5510,10 +6074,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5522,10 +6086,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5534,10 +6098,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5546,10 +6110,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -5558,17 +6122,17 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5596,13 +6160,66 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -5612,13 +6229,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5635,7 +6252,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ], "fineGrainedLineages": [ @@ -5646,10 +6263,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5658,10 +6275,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5670,10 +6287,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5682,10 +6299,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5694,10 +6311,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5706,10 +6323,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5718,10 +6335,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5730,10 +6347,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5742,10 +6359,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5754,23 +6371,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_5\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -5780,44 +6425,44 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5825,33 +6470,21 @@ "entityType": "query", "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_5\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "platform": "urn:li:dataPlatform:snowflake" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5868,7 +6501,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ], "fineGrainedLineages": [ @@ -5879,10 +6512,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5891,10 +6524,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5903,10 +6536,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5915,10 +6548,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5927,10 +6560,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5939,10 +6572,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5951,10 +6584,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5963,10 +6596,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5975,10 +6608,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5987,23 +6620,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_6\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -6013,66 +6674,50 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -6082,13 +6727,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6105,7 +6750,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ], "fineGrainedLineages": [ @@ -6116,10 +6761,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6128,10 +6773,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6140,10 +6785,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6152,10 +6797,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6164,10 +6809,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6176,10 +6821,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6188,10 +6833,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6200,10 +6845,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6212,10 +6857,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6224,29 +6869,29 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_3\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_7\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", "language": "SQL" }, "source": "SYSTEM", @@ -6262,13 +6907,66 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -6278,13 +6976,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6301,7 +6999,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ], "fineGrainedLineages": [ @@ -6312,10 +7010,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6324,10 +7022,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6336,10 +7034,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6348,10 +7046,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6360,10 +7058,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6372,10 +7070,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6384,10 +7082,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6396,10 +7094,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6408,10 +7106,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6420,23 +7118,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_8\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -6446,50 +7172,66 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6506,7 +7248,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" } ], "fineGrainedLineages": [ @@ -6517,10 +7259,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6529,10 +7271,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6541,10 +7283,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6553,10 +7295,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6565,10 +7307,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6577,10 +7319,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6589,10 +7331,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -6601,214 +7343,53 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" - ], - "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" - ], - "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + } + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_6\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_9\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", "language": "SQL" }, "source": "SYSTEM", @@ -6824,58 +7405,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -6885,44 +7421,44 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -6938,93 +7474,270 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", + "aspectName": "upstreamLineage", "aspect": { "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] + "upstreams": [ + { + "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + } + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_8\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "value": "CREATE VIEW view_1 AS\nSELECT\n *\nFROM table_1", "language": "SQL" }, "source": "SYSTEM", "created": { - "time": 1654473600000, + "time": 0, "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1654473600000, + "time": 1738040516852, "actor": "urn:li:corpuser:_ingestion" } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", + "aspectName": "querySubjects", "aspect": { "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + } + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7032,163 +7745,278 @@ "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "statement": { - "value": "CREATE VIEW view_1 AS\nSELECT\n *\nFROM table_1", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1710193011317, - "actor": "urn:li:corpuser:_ingestion" - } + "platform": "urn:li:dataPlatform:snowflake" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "upstreamLineage", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, + "upstreams": [ { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" - }, + "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + } + ], + "fineGrainedLineages": [ { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW view_2 AS\nSELECT\n *\nFROM table_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1738040516859, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7204,41 +8032,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_9\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -7260,77 +8089,66 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "platform": "urn:li:dataPlatform:snowflake" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7358,36 +8176,37 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_7\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7415,178 +8234,159 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "CREATE VIEW view_2 AS\nSELECT\n *\nFROM table_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1710193011336, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -7608,8 +8408,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7620,7 +8420,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7631,8 +8431,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7643,7 +8443,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7654,8 +8454,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7666,7 +8466,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7677,8 +8477,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7689,7 +8489,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7700,8 +8500,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7712,7 +8512,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7723,8 +8523,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7735,7 +8535,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7746,8 +8546,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7758,7 +8558,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7769,8 +8569,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7781,7 +8581,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7792,8 +8592,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7804,7 +8604,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7815,8 +8615,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7827,7 +8627,7 @@ "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -7838,14 +8638,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7854,14 +8654,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7870,14 +8670,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7886,14 +8686,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7902,14 +8702,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7918,14 +8718,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7935,13 +8735,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7951,13 +8751,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7967,13 +8767,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7983,13 +8783,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -7999,13 +8799,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8015,13 +8815,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8031,13 +8831,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityType": "tag", + "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8047,13 +8847,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8063,13 +8863,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8079,13 +8879,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8095,13 +8895,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8111,7 +8911,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -8127,39 +8927,39 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "glossaryTerm", - "entityUrn": "urn:li:glossaryTerm:Email_Address", + "entityUrn": "urn:li:glossaryTerm:CloudRegion", "changeType": "UPSERT", "aspectName": "glossaryTermKey", "aspect": { "json": { - "name": "Email_Address" + "name": "CloudRegion" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "glossaryTerm", - "entityUrn": "urn:li:glossaryTerm:CloudRegion", + "entityUrn": "urn:li:glossaryTerm:Email_Address", "changeType": "UPSERT", "aspectName": "glossaryTermKey", "aspect": { "json": { - "name": "CloudRegion" + "name": "Email_Address" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index f22cbd122361dc..87eca14c7655eb 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -4101,6 +4101,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)", @@ -4117,6 +4133,344 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "PATCH", + "aspectName": "datasetProperties", + "aspect": { + "json": [ + { + "op": "add", + "path": "/name", + "value": "STREAM_1" + }, + { + "op": "add", + "path": "/description", + "value": "Comment for Stream 1" + }, + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.STREAM_1" + }, + { + "op": "add", + "path": "/customProperties/SOURCE_TYPE", + "value": "Table" + }, + { + "op": "add", + "path": "/customProperties/TYPE", + "value": "DELTA" + }, + { + "op": "add", + "path": "/customProperties/STALE", + "value": "false" + }, + { + "op": "add", + "path": "/customProperties/MODE", + "value": "DEFAULT" + }, + { + "op": "add", + "path": "/customProperties/OWNER_ROLE_TYPE", + "value": "ROLE" + }, + { + "op": "add", + "path": "/customProperties/TABLE_NAME", + "value": "TEST_DB.TEST_SCHEMA.TABLE_1" + }, + { + "op": "add", + "path": "/customProperties/BASE_TABLES", + "value": "TEST_DB.TEST_SCHEMA.TABLE_1" + }, + { + "op": "add", + "path": "/customProperties/STALE_AFTER", + "value": "2021-06-22T00:00:00+00:00" + } + ] + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-ftc9zc", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", @@ -4396,6 +4750,35 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)" + }, + { + "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab", + "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab" + }, + { + "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f", + "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json index 3adedd59070396..559856e1f654b3 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json @@ -3729,6 +3729,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", @@ -3881,6 +3897,279 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00+00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623110400000 + }, + "lastModified": { + "time": 1623110400000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_17-13_30_28-vz7bmd", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", @@ -4008,6 +4297,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index d2e20e784282ee..f94627b1c22f9c 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -180,7 +180,8 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): cache_info = report.data_dictionary_cache.as_obj() assert cache_info["get_tables_for_database"]["misses"] == 1 assert cache_info["get_views_for_database"]["misses"] == 1 - assert cache_info["get_columns_for_schema"]["misses"] == 1 + # When streams query specific tables, the query will not be cached resulting in 2 cache misses + assert cache_info["get_columns_for_schema"]["misses"] == 2 assert cache_info["get_pk_constraints_for_schema"]["misses"] == 1 assert cache_info["get_fk_constraints_for_schema"]["misses"] == 1 diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index 4cb6cec4906efa..808734c13801ec 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -143,12 +143,16 @@ def test_snowflake_no_tables_causes_pipeline_failure( [SnowflakeQuery.tables_for_schema("TEST_SCHEMA", "TEST_DB")], [], ) - sf_cursor.execute.side_effect = query_permission_response_override( + no_views_fn = query_permission_response_override( no_tables_fn, [SnowflakeQuery.show_views_for_database("TEST_DB")], [], ) - + sf_cursor.execute.side_effect = query_permission_response_override( + no_views_fn, + [SnowflakeQuery.streams_for_database("TEST_DB")], + [], + ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() assert "permission-error" in [