From eab2ac7a2e708e213d75c245589978d2b2aec246 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Wed, 23 Oct 2024 14:02:08 +0530 Subject: [PATCH 01/26] =?UTF-8?q?feat(ingest/snowflake):=20support=20linea?= =?UTF-8?q?ge=20via=20rename=20and=20swap=20using=20que=E2=80=A6=20(#11600?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ingestion/source/redshift/lineage.py | 19 +- .../ingestion/source/redshift/lineage_v2.py | 6 +- .../source/snowflake/snowflake_queries.py | 75 ++- .../sql_parsing/sql_parsing_aggregator.py | 169 ++++-- .../aggregator_goldens/test_table_rename.json | 236 ++++++++- .../test_table_rename_with_temp.json | 303 +++++++++++ .../aggregator_goldens/test_table_swap.json | 490 ++++++++++++++++++ .../test_table_swap_with_temp.json | 271 ++++++++++ ...t_table_swap_with_temp_with_preparsed.json | 298 +++++++++++ .../unit/sql_parsing/test_sql_aggregator.py | 251 ++++++++- 10 files changed, 2037 insertions(+), 81 deletions(-) create mode 100644 metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index fe491ccb31850..192c97bbee348 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -53,6 +53,7 @@ ) from datahub.metadata.urns import DatasetUrn from datahub.sql_parsing.schema_resolver import SchemaResolver +from datahub.sql_parsing.sql_parsing_aggregator import TableRename from datahub.sql_parsing.sqlglot_utils import get_dialect, parse_statement from datahub.utilities import memory_footprint from datahub.utilities.dedup_list import deduplicate_list @@ -504,21 +505,21 @@ def _populate_lineage_map( self.report_status(f"extract-{lineage_type.name}", False) def _update_lineage_map_for_table_renames( - self, table_renames: Dict[str, str] + self, table_renames: Dict[str, TableRename] ) -> None: if not table_renames: return logger.info(f"Updating lineage map for {len(table_renames)} table renames") - for new_table_urn, prev_table_urn in table_renames.items(): + for entry in table_renames.values(): # This table was renamed from some other name, copy in the lineage # for the previous name as well. - prev_table_lineage = self._lineage_map.get(prev_table_urn) + prev_table_lineage = self._lineage_map.get(entry.original_urn) if prev_table_lineage: logger.debug( - f"including lineage for {prev_table_urn} in {new_table_urn} due to table rename" + f"including lineage for {entry.original_urn} in {entry.new_urn} due to table rename" ) - self._lineage_map[new_table_urn].merge_lineage( + self._lineage_map[entry.new_urn].merge_lineage( upstreams=prev_table_lineage.upstreams, cll=prev_table_lineage.cll, ) @@ -672,7 +673,7 @@ def populate_lineage( for db, schemas in all_tables.items() } - table_renames: Dict[str, str] = {} + table_renames: Dict[str, TableRename] = {} if self.config.include_table_rename_lineage: table_renames, all_tables_set = self._process_table_renames( database=database, @@ -851,11 +852,11 @@ def _process_table_renames( database: str, connection: redshift_connector.Connection, all_tables: Dict[str, Dict[str, Set[str]]], - ) -> Tuple[Dict[str, str], Dict[str, Dict[str, Set[str]]]]: + ) -> Tuple[Dict[str, TableRename], Dict[str, Dict[str, Set[str]]]]: logger.info(f"Processing table renames for db {database}") # new urn -> prev urn - table_renames: Dict[str, str] = {} + table_renames: Dict[str, TableRename] = {} query = self.queries.alter_table_rename_query( db_name=database, @@ -893,7 +894,7 @@ def _process_table_renames( env=self.config.env, ) - table_renames[new_urn] = prev_urn + table_renames[new_urn] = TableRename(prev_urn, new_urn, query_text) # We want to generate lineage for the previous name too. all_tables[database][schema].add(prev_name) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py index 53f9383ec02a7..e7cc5fc50cf6e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage_v2.py @@ -146,10 +146,8 @@ def build( lambda: collections.defaultdict(set) ), ) - for new_urn, original_urn in table_renames.items(): - self.aggregator.add_table_rename( - original_urn=original_urn, new_urn=new_urn - ) + for entry in table_renames.values(): + self.aggregator.add_table_rename(entry) if self.config.table_lineage_mode in { LineageMode.SQL_BASED, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index e11073d77b46e..33dc1388ff0e1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -52,6 +52,8 @@ PreparsedQuery, SqlAggregatorReport, SqlParsingAggregator, + TableRename, + TableSwap, ) from datahub.sql_parsing.sql_parsing_common import QueryType from datahub.sql_parsing.sqlglot_lineage import ( @@ -116,6 +118,8 @@ class SnowflakeQueriesExtractorReport(Report): audit_log_load_timer: PerfTimer = dataclasses.field(default_factory=PerfTimer) sql_aggregator: Optional[SqlAggregatorReport] = None + num_ddl_queries_dropped: int = 0 + @dataclass class SnowflakeQueriesSourceReport(SourceReport): @@ -225,7 +229,9 @@ def get_workunits_internal( audit_log_file = self.local_temp_path / "audit_log.sqlite" use_cached_audit_log = audit_log_file.exists() - queries: FileBackedList[Union[KnownLineageMapping, PreparsedQuery]] + queries: FileBackedList[ + Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] + ] if use_cached_audit_log: logger.info("Using cached audit log") shared_connection = ConnectionWrapper(audit_log_file) @@ -235,7 +241,7 @@ def get_workunits_internal( shared_connection = ConnectionWrapper(audit_log_file) queries = FileBackedList(shared_connection) - entry: Union[KnownLineageMapping, PreparsedQuery] + entry: Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] with self.report.copy_history_fetch_timer: for entry in self.fetch_copy_history(): @@ -296,7 +302,7 @@ def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: def fetch_query_log( self, - ) -> Iterable[PreparsedQuery]: + ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap]]: query_log_query = _build_enriched_query_log_query( start_time=self.config.window.start_time, end_time=self.config.window.end_time, @@ -324,12 +330,16 @@ def fetch_query_log( exc=e, ) else: - yield entry + if entry: + yield entry - def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery: + def _parse_audit_log_row( + self, row: Dict[str, Any] + ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery]]: json_fields = { "DIRECT_OBJECTS_ACCESSED", "OBJECTS_MODIFIED", + "OBJECT_MODIFIED_BY_DDL", } res = {} @@ -341,6 +351,17 @@ def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery: direct_objects_accessed = res["direct_objects_accessed"] objects_modified = res["objects_modified"] + object_modified_by_ddl = res["object_modified_by_ddl"] + + if object_modified_by_ddl and not objects_modified: + ddl_entry: Optional[Union[TableRename, TableSwap]] = None + with self.structured_reporter.report_exc( + "Error fetching ddl lineage from Snowflake" + ): + ddl_entry = self.parse_ddl_query( + res["query_text"], object_modified_by_ddl + ) + return ddl_entry upstreams = [] column_usage = {} @@ -437,6 +458,45 @@ def _parse_audit_log_row(self, row: Dict[str, Any]) -> PreparsedQuery: ) return entry + def parse_ddl_query( + self, query: str, object_modified_by_ddl: dict + ) -> Optional[Union[TableRename, TableSwap]]: + if object_modified_by_ddl[ + "operationType" + ] == "ALTER" and object_modified_by_ddl["properties"].get("swapTargetName"): + urn1 = self.identifiers.gen_dataset_urn( + self.identifiers.get_dataset_identifier_from_qualified_name( + object_modified_by_ddl["objectName"] + ) + ) + + urn2 = self.identifiers.gen_dataset_urn( + self.identifiers.get_dataset_identifier_from_qualified_name( + object_modified_by_ddl["properties"]["swapTargetName"]["value"] + ) + ) + + return TableSwap(urn1, urn2, query) + elif object_modified_by_ddl[ + "operationType" + ] == "RENAME_TABLE" and object_modified_by_ddl["properties"].get("objectName"): + original_un = self.identifiers.gen_dataset_urn( + self.identifiers.get_dataset_identifier_from_qualified_name( + object_modified_by_ddl["objectName"] + ) + ) + + new_urn = self.identifiers.gen_dataset_urn( + self.identifiers.get_dataset_identifier_from_qualified_name( + object_modified_by_ddl["properties"]["objectName"]["value"] + ) + ) + + return TableRename(original_un, new_urn, query) + else: + self.report.num_ddl_queries_dropped += 1 + return None + def close(self) -> None: self._exit_stack.close() @@ -542,6 +602,7 @@ def _build_enriched_query_log_query( user_name, direct_objects_accessed, objects_modified, + object_modified_by_ddl FROM snowflake.account_usage.access_history WHERE @@ -563,8 +624,9 @@ def _build_enriched_query_log_query( ) as direct_objects_accessed, -- TODO: Drop the columns.baseSources subfield. FILTER(objects_modified, o -> o:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER}) as objects_modified, + case when object_modified_by_ddl:objectDomain IN {SnowflakeQuery.ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER} then object_modified_by_ddl else null end as object_modified_by_ddl FROM raw_access_history - WHERE ( array_size(direct_objects_accessed) > 0 or array_size(objects_modified) > 0 ) + WHERE ( array_size(direct_objects_accessed) > 0 or array_size(objects_modified) > 0 or object_modified_by_ddl is not null ) ) , query_access_history AS ( SELECT @@ -586,6 +648,7 @@ def _build_enriched_query_log_query( q.role_name AS "ROLE_NAME", a.direct_objects_accessed, a.objects_modified, + a.object_modified_by_ddl FROM deduplicated_queries q JOIN filtered_access_history a USING (query_id) ) diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 0b7ad14a8c1b4..9aab33bf09f05 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -174,6 +174,26 @@ class KnownLineageMapping: lineage_type: str = models.DatasetLineageTypeClass.COPY +@dataclasses.dataclass +class TableRename: + original_urn: UrnStr + new_urn: UrnStr + query: Optional[str] = None + session_id: str = _MISSING_SESSION_ID + + +@dataclasses.dataclass +class TableSwap: + urn1: UrnStr + urn2: UrnStr + query: Optional[str] = None + session_id: str = _MISSING_SESSION_ID + + def id(self) -> str: + # TableSwap(A,B) is same as TableSwap(B,A) + return str(hash(frozenset([self.urn1, self.urn2]))) + + @dataclasses.dataclass class PreparsedQuery: # If not provided, we will generate one using the fast fingerprint generator. @@ -237,6 +257,7 @@ class SqlAggregatorReport(Report): num_preparsed_queries: int = 0 num_known_mapping_lineage: int = 0 num_table_renames: int = 0 + num_table_swaps: int = 0 # Temp tables. num_temp_sessions: Optional[int] = None @@ -442,6 +463,12 @@ def __init__( ) self._exit_stack.push(self._table_renames) + # Map of table swaps, from unique swap id to TableSwap + self._table_swaps = FileBackedDict[TableSwap]( + shared_connection=self._shared_connection, tablename="table_swaps" + ) + self._exit_stack.push(self._table_swaps) + # Usage aggregator. This will only be initialized if usage statistics are enabled. # TODO: Replace with FileBackedDict. # TODO: The BaseUsageConfig class is much too broad for our purposes, and has a number of @@ -533,7 +560,12 @@ def is_allowed_table(self, urn: UrnStr) -> bool: def add( self, item: Union[ - KnownQueryLineageInfo, KnownLineageMapping, PreparsedQuery, ObservedQuery + KnownQueryLineageInfo, + KnownLineageMapping, + PreparsedQuery, + ObservedQuery, + TableRename, + TableSwap, ], ) -> None: if isinstance(item, KnownQueryLineageInfo): @@ -544,6 +576,10 @@ def add( self.add_preparsed_query(item) elif isinstance(item, ObservedQuery): self.add_observed_query(item) + elif isinstance(item, TableRename): + self.add_table_rename(item) + elif isinstance(item, TableSwap): + self.add_table_swap(item) else: raise ValueError(f"Cannot add unknown item type: {type(item)}") @@ -629,19 +665,11 @@ def add_known_lineage_mapping( query_id = self._known_lineage_query_id() # Generate CLL if schema of downstream is known - column_lineage: List[ColumnLineageInfo] = [] - if self._schema_resolver.has_urn(downstream_urn): - schema = self._schema_resolver._resolve_schema_info(downstream_urn) - if schema: - column_lineage = [ - ColumnLineageInfo( - downstream=DownstreamColumnRef( - table=downstream_urn, column=field_path - ), - upstreams=[ColumnRef(table=upstream_urn, column=field_path)], - ) - for field_path in schema - ] + column_lineage: List[ + ColumnLineageInfo + ] = self._generate_identity_column_lineage( + upstream_urn=upstream_urn, downstream_urn=downstream_urn + ) # Register the query. self._add_to_query_map( @@ -663,6 +691,25 @@ def add_known_lineage_mapping( # Register the lineage. self._lineage_map.for_mutation(downstream_urn, OrderedSet()).add(query_id) + def _generate_identity_column_lineage( + self, *, upstream_urn: UrnStr, downstream_urn: UrnStr + ) -> List[ColumnLineageInfo]: + column_lineage: List[ColumnLineageInfo] = [] + if self._schema_resolver.has_urn(downstream_urn): + schema = self._schema_resolver._resolve_schema_info(downstream_urn) + if schema: + column_lineage = [ + ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=downstream_urn, column=field_path + ), + upstreams=[ColumnRef(table=upstream_urn, column=field_path)], + ) + for field_path in schema + ] + + return column_lineage + def add_view_definition( self, view_urn: Union[DatasetUrn, UrnStr], @@ -849,12 +896,6 @@ def add_preparsed_query( return out_table = parsed.downstream - # Handle table renames. - is_renamed_table = False - if out_table in self._table_renames: - out_table = self._table_renames[out_table] - is_renamed_table = True - # Register the query's lineage. if ( is_known_temp_table @@ -863,13 +904,10 @@ def add_preparsed_query( and parsed.query_type_props.get("temporary") ) or ( - not is_renamed_table - and ( - self.is_temp_table(out_table) - or ( - require_out_table_schema - and not self._schema_resolver.has_urn(out_table) - ) + self.is_temp_table(out_table) + or ( + require_out_table_schema + and not self._schema_resolver.has_urn(out_table) ) ) ): @@ -896,26 +934,81 @@ def add_preparsed_query( def add_table_rename( self, - original_urn: UrnStr, - new_urn: UrnStr, + table_rename: TableRename, ) -> None: """Add a table rename to the aggregator. - This will so that all _future_ observed queries that reference the original urn - will instead generate usage and lineage for the new urn. + This will make all observed queries that reference the original urn + will instead generate lineage for the new urn. + """ + + self.report.num_table_renames += 1 + + # This will not work if the table is renamed multiple times. + self._table_renames[table_rename.original_urn] = table_rename.new_urn - Currently, this does not affect any queries that have already been observed. - TODO: Add a mechanism to update the lineage for queries that have already been observed. + original_table = self._name_from_urn(table_rename.original_urn) + new_table = self._name_from_urn(table_rename.new_urn) + + self.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text=table_rename.query + or f"--Datahub generated query text--\n" + f"alter table {original_table} rename to {new_table}", + upstreams=[table_rename.original_urn], + downstream=table_rename.new_urn, + column_lineage=self._generate_identity_column_lineage( + downstream_urn=table_rename.new_urn, + upstream_urn=table_rename.original_urn, + ), + ) + ) + + def add_table_swap(self, table_swap: TableSwap) -> None: + """Add a table swap to the aggregator. Args: - original_urn: The original dataset URN. - new_urn: The new dataset URN. + table_swap.urn1, table_swap.urn2: The dataset URNs to swap. """ - self.report.num_table_renames += 1 + if table_swap.id() in self._table_swaps: + # We have already processed this table swap once + return - # This will not work if the table is renamed multiple times. - self._table_renames[original_urn] = new_urn + self.report.num_table_swaps += 1 + self._table_swaps[table_swap.id()] = table_swap + table1 = self._name_from_urn(table_swap.urn1) + table2 = self._name_from_urn(table_swap.urn2) + + # NOTE: Both queries are different on purpose. Currently, we can not + # store (A->B) and (B->A) lineage against same query. + + # NOTE: we do not store upstreams for temp table on purpose, as that would + # otherwise overwrite original upstream query of temp table because + # currently a temporay table can have only one upstream query. + + if not self.is_temp_table(table_swap.urn2): + self.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text=f"--Datahub generated query text--" + f"\nalter table {table1} swap with {table2}", + upstreams=[table_swap.urn1], + downstream=table_swap.urn2, + ) + ) + + if not self.is_temp_table(table_swap.urn1): + self.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text=f"--Datahub generated query text--\n" + f"alter table {table2} swap with {table1}", + upstreams=[table_swap.urn2], + downstream=table_swap.urn1, + ) + ) def _make_schema_resolver_for_session( self, session_id: str diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json index 9a4d405e50a7a..19d19d1f56ae9 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json @@ -119,6 +119,198 @@ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "ALTER TABLE dev.public.foo_staging RENAME TO foo", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE foo_staging AS\nSELECT\n a,\n b\nFROM foo_dep", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d" + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", "aspect": { "json": { "upstreams": [ @@ -144,7 +336,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)" ], "confidenceScore": 0.2, "query": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc" @@ -156,7 +348,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)" ], "confidenceScore": 0.2, "query": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc" @@ -167,13 +359,13 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "CREATE TABLE foo_staging AS\nSELECT\n a,\n b\nFROM foo_dep", + "value": "CREATE TABLE foo_downstream AS\nSELECT\n a,\n b\nFROM foo_staging", "language": "SQL" }, "source": "SYSTEM", @@ -190,43 +382,43 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e4b3b60ab99e0f0bc1629ea82a5d7705a30dbd98a3923d599b39fb68624ea58d", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)" } ] } } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:redshift" - } - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json new file mode 100644 index 0000000000000..4e2eba778ab9a --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json @@ -0,0 +1,303 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE bar AS\nSELECT\n a,\n b\nFROM baz", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.baz,PROD),a)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD),b)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE foo_staging AS\nSELECT\n a,\n b\nFROM foo_dep;\n\nALTER TABLE dev.public.foo_staging RENAME TO foo", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE foo_staging AS\nSELECT\n a,\n b\nFROM foo_dep;\n\nCREATE TABLE foo_downstream AS\nSELECT\n a,\n b\nFROM foo_staging", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json new file mode 100644 index 0000000000000..382bc8ee0281d --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json @@ -0,0 +1,490 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE bar AS\nSELECT\n a,\n b\nFROM baz", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "ALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE person_info_backup AS\nSELECT\n *\nFROM person_info_swap", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68" + }, + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:4b1fad909083e1ed5c47c146bd01247ed4d6295d175c34f9065b8fc6000fc7ae", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "ALTER TABLE dev.public.person_info SWAP WITH dev.public.person_info_swap", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + } + ] + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json new file mode 100644 index 0000000000000..73084f49a8928 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json @@ -0,0 +1,271 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE bar AS\nSELECT\n a,\n b\nFROM baz", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.baz,PROD),b)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.bar,PROD),b)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nCREATE TABLE person_info_backup AS\nSELECT\n *\nFROM person_info_swap", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json new file mode 100644 index 0000000000000..24ed6a3b54d8c --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp_with_preparsed.json @@ -0,0 +1,298 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "ALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE TABLE person_info_backup AS\nSELECT\n *\nFROM person_info_swap", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:6f71602f39d01a39b3f8bd411c74c5ac08dc4b90bc3d49b257089acb19fa8559", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68" + }, + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "ALTER TABLE dev.public.person_info SWAP WITH dev.public.person_info_swap", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d29a1c8ed6d4d77efb290260234e5eee56f98311a5631d0a12213798077d1a68", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + } + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_incremental,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:481d0392ffeffdafd198d94e0a9f778dd722b60daa47083a32800b99ea21f86f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py index 849d550ef69c5..45bbc19a2124f 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py @@ -13,8 +13,11 @@ from datahub.sql_parsing.sql_parsing_aggregator import ( KnownQueryLineageInfo, ObservedQuery, + PreparsedQuery, QueryLogSetting, SqlParsingAggregator, + TableRename, + TableSwap, ) from datahub.sql_parsing.sql_parsing_common import QueryType from datahub.sql_parsing.sqlglot_lineage import ( @@ -522,8 +525,10 @@ def test_table_rename(pytestconfig: pytest.Config) -> None: # Register that foo_staging is renamed to foo. aggregator.add_table_rename( - original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(), - new_urn=DatasetUrn("redshift", "dev.public.foo").urn(), + TableRename( + original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(), + new_urn=DatasetUrn("redshift", "dev.public.foo").urn(), + ) ) # Add an unrelated query. @@ -544,6 +549,15 @@ def test_table_rename(pytestconfig: pytest.Config) -> None: ) ) + # Add the query that created the downstream from foo_staging table. + aggregator.add_observed_query( + ObservedQuery( + query="create table foo_downstream as select a, b from foo_staging", + default_db="dev", + default_schema="public", + ) + ) + mcps = list(aggregator.gen_metadata()) mce_helpers.check_goldens_stream( @@ -553,6 +567,226 @@ def test_table_rename(pytestconfig: pytest.Config) -> None: ) +@freeze_time(FROZEN_TIME) +def test_table_rename_with_temp(pytestconfig: pytest.Config) -> None: + aggregator = SqlParsingAggregator( + platform="redshift", + generate_lineage=True, + generate_usage_statistics=False, + generate_operations=False, + is_temp_table=lambda x: "staging" in x.lower(), + ) + + # Register that foo_staging is renamed to foo. + aggregator.add_table_rename( + TableRename( + original_urn=DatasetUrn("redshift", "dev.public.foo_staging").urn(), + new_urn=DatasetUrn("redshift", "dev.public.foo").urn(), + query="alter table dev.public.foo_staging rename to dev.public.foo", + ) + ) + + # Add an unrelated query. + aggregator.add_observed_query( + ObservedQuery( + query="create table bar as select a, b from baz", + default_db="dev", + default_schema="public", + ) + ) + + # Add the query that created the staging table. + aggregator.add_observed_query( + ObservedQuery( + query="create table foo_staging as select a, b from foo_dep", + default_db="dev", + default_schema="public", + ) + ) + + # Add the query that created the downstream from foo_staging table. + aggregator.add_observed_query( + ObservedQuery( + query="create table foo_downstream as select a, b from foo_staging", + default_db="dev", + default_schema="public", + ) + ) + + mcps = list(aggregator.gen_metadata()) + + mce_helpers.check_goldens_stream( + pytestconfig, + outputs=mcps, + golden_path=RESOURCE_DIR / "test_table_rename_with_temp.json", + ) + + +@freeze_time(FROZEN_TIME) +def test_table_swap(pytestconfig: pytest.Config) -> None: + aggregator = SqlParsingAggregator( + platform="snowflake", + generate_lineage=True, + generate_usage_statistics=False, + generate_operations=False, + ) + + # Add an unrelated query. + aggregator.add_observed_query( + ObservedQuery( + query="create table bar as select a, b from baz", + default_db="dev", + default_schema="public", + ) + ) + + # Add the query that created the swap table initially. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="CREATE TABLE person_info_swap CLONE person_info;", + upstreams=[DatasetUrn("snowflake", "dev.public.person_info").urn()], + downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + # Add the query that created the incremental table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="CREATE TABLE person_info_incremental AS SELECT * from person_info_dep;", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_dep").urn(), + ], + downstream=DatasetUrn( + "snowflake", "dev.public.person_info_incremental" + ).urn(), + ) + ) + + # Add the query that updated the swap table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="INSERT INTO person_info_swap SELECT * from person_info_incremental;", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_incremental").urn(), + ], + downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + aggregator.add_table_swap( + TableSwap( + urn1=DatasetUrn("snowflake", "dev.public.person_info").urn(), + urn2=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + # Add the query that is created from swap table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="create table person_info_backup as select * from person_info_swap", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ], + downstream=DatasetUrn("snowflake", "dev.public.person_info_backup").urn(), + ) + ) + + mcps = list(aggregator.gen_metadata()) + + mce_helpers.check_goldens_stream( + pytestconfig, + outputs=mcps, + golden_path=RESOURCE_DIR / "test_table_swap.json", + ) + + +@freeze_time(FROZEN_TIME) +def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: + aggregator = SqlParsingAggregator( + platform="snowflake", + generate_lineage=True, + generate_usage_statistics=False, + generate_operations=False, + is_temp_table=lambda x: "swap" in x.lower() or "incremental" in x.lower(), + ) + + # Add an unrelated query. + aggregator.add_observed_query( + ObservedQuery( + query="create table bar as select a, b from baz", + default_db="dev", + default_schema="public", + ) + ) + + # Add the query that created the swap table initially. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="CREATE TABLE person_info_swap CLONE person_info;", + upstreams=[DatasetUrn("snowflake", "dev.public.person_info").urn()], + downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + # Add the query that created the incremental table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="CREATE TABLE person_info_incremental AS SELECT * from person_info_dep;", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_dep").urn(), + ], + downstream=DatasetUrn( + "snowflake", "dev.public.person_info_incremental" + ).urn(), + ) + ) + + # Add the query that updated the swap table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="INSERT INTO person_info_swap SELECT * from person_info_incremental;", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_incremental").urn(), + ], + downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + aggregator.add_table_swap( + TableSwap( + urn1=DatasetUrn("snowflake", "dev.public.person_info").urn(), + urn2=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ) + ) + + # Add the query that is created from swap table. + aggregator.add_preparsed_query( + PreparsedQuery( + query_id=None, + query_text="create table person_info_backup as select * from person_info_swap", + upstreams=[ + DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + ], + downstream=DatasetUrn("snowflake", "dev.public.person_info_backup").urn(), + ) + ) + + mcps = list(aggregator.gen_metadata()) + + mce_helpers.check_goldens_stream( + pytestconfig, + outputs=mcps, + golden_path=RESOURCE_DIR / "test_table_swap_with_temp.json", + ) + + @freeze_time(FROZEN_TIME) def test_create_table_query_mcps(pytestconfig: pytest.Config) -> None: aggregator = SqlParsingAggregator( @@ -665,6 +899,19 @@ def test_basic_usage(pytestconfig: pytest.Config) -> None: ) +def test_table_swap_id() -> None: + assert ( + TableSwap( + urn1=DatasetUrn("snowflake", "dev.public.foo").urn(), + urn2=DatasetUrn("snowflake", "dev.public.foo_staging").urn(), + ).id() + == TableSwap( + urn1=DatasetUrn("snowflake", "dev.public.foo_staging").urn(), + urn2=DatasetUrn("snowflake", "dev.public.foo").urn(), + ).id() + ) + + def test_sql_aggreator_close_cleans_tmp(tmp_path): frozen_timestamp = parse_user_datetime(FROZEN_TIME) with patch("tempfile.tempdir", str(tmp_path)): From e569fbc8a6f7023bed6341e82806d003054d2a29 Mon Sep 17 00:00:00 2001 From: pankajmahato-visa <154867659+pankajmahato-visa@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:40:20 +0530 Subject: [PATCH 02/26] fix(ingest/mongodb): Add Collection Name as Dataset Name in MongoDB (#11698) --- metadata-ingestion/src/datahub/ingestion/source/mongodb.py | 1 + .../tests/integration/mongodb/mongodb_mces_golden.json | 4 ++++ .../mongodb/mongodb_mces_small_schema_size_golden.json | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index e3fd8849ed844..c87b025f13b55 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -410,6 +410,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) dataset_properties = DatasetPropertiesClass( + name=collection_name, tags=[], customProperties={}, ) diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json index 2666990ed1333..103977463a8f2 100644 --- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json +++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json @@ -93,6 +93,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "emptyCollection", "customProperties": {}, "tags": [] } @@ -503,6 +504,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "firstCollection", "customProperties": {}, "tags": [] } @@ -4201,6 +4203,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "largeCollection", "customProperties": { "schema.downsampled": "True", "schema.totalFields": "501" @@ -4444,6 +4447,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "secondCollection", "customProperties": {}, "tags": [] } diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json index 577a053f1c741..5cf1ff47a3866 100644 --- a/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json +++ b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_small_schema_size_golden.json @@ -93,6 +93,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "emptyCollection", "customProperties": {}, "tags": [] } @@ -359,6 +360,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "firstCollection", "customProperties": { "schema.downsampled": "True", "schema.totalFields": "22" @@ -580,6 +582,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "largeCollection", "customProperties": { "schema.downsampled": "True", "schema.totalFields": "502" @@ -823,6 +826,7 @@ "aspectName": "datasetProperties", "aspect": { "json": { + "name": "secondCollection", "customProperties": {}, "tags": [] } From a1d4a7493655f2a6df1c0bfde64a68ab105f7ccb Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 23 Oct 2024 12:39:14 -0700 Subject: [PATCH 03/26] fix(ui): show structured report context in pre (#11673) --- .../source/executions/reporting/StructuredReportItemContext.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx index abf6ce2c676bc..91b6282088f46 100644 --- a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx +++ b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx @@ -17,7 +17,7 @@ const Title = styled.div` font-weight: bold; `; -const Item = styled.div` +const Item = styled.pre` padding: 6px; font-size: 12px; border-radius: 2px; From 971aae076d1347ad81f0ea2e757ef0b31ad64bbd Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 23 Oct 2024 15:29:00 -0500 Subject: [PATCH 04/26] feat(logs): add change event details to log context and improve some logs in MCL/MCP (#11690) --- .../java/com/linkedin/metadata/Constants.java | 6 ++++ .../service/UpdateGraphIndicesService.java | 17 ++++++++--- .../service/UpdateIndicesService.java | 23 +++++++++++---- .../src/main/resources/logback.xml | 2 +- .../metadata/kafka/MCLKafkaListener.java | 29 ++++++++++++++++--- .../src/main/resources/logback.xml | 2 +- .../MetadataChangeProposalsProcessor.java | 21 ++++++++++++++ 7 files changed, 85 insertions(+), 15 deletions(-) diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 8961677b56878..c280324e10f47 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -458,5 +458,11 @@ public class Constants { // DAO public static final long LATEST_VERSION = 0; + // Logging MDC + public static final String MDC_ENTITY_URN = "entityUrn"; + public static final String MDC_ASPECT_NAME = ""; + public static final String MDC_ENTITY_TYPE = "entityType"; + public static final String MDC_CHANGE_TYPE = "changeType"; + private Constants() {} } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java index 7549aea2007da..3829855caa08b 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java @@ -34,6 +34,7 @@ import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.utils.SchemaFieldUtils; +import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; @@ -56,6 +57,9 @@ @Slf4j public class UpdateGraphIndicesService implements SearchIndicesService { private static final String DOWNSTREAM_OF = "DownstreamOf"; + private static final String GRAPH_DIFF_MODE_REMOVE_METRIC = "diff_remove_edge"; + private static final String GRAPH_DIFF_MODE_ADD_METRIC = "diff_add_edge"; + private static final String GRAPH_DIFF_MODE_UPDATE_METRIC = "diff_update_edge"; public static UpdateGraphIndicesService withService(GraphService graphService) { return new UpdateGraphIndicesService(graphService); @@ -382,20 +386,25 @@ private void updateGraphServiceDiff( final List mergedEdges = getMergedEdges(oldEdgeSet, newEdgeSet); // Remove any old edges that no longer exist first - if (subtractiveDifference.size() > 0) { + if (!subtractiveDifference.isEmpty()) { log.debug("Removing edges: {}", subtractiveDifference); + MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_REMOVE_METRIC) + .inc(subtractiveDifference.size()); subtractiveDifference.forEach(graphService::removeEdge); } // Then add new edges - if (additiveDifference.size() > 0) { + if (!additiveDifference.isEmpty()) { log.debug("Adding edges: {}", additiveDifference); + MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_ADD_METRIC) + .inc(additiveDifference.size()); additiveDifference.forEach(graphService::addEdge); } // Then update existing edges - if (mergedEdges.size() > 0) { + if (!mergedEdges.isEmpty()) { log.debug("Updating edges: {}", mergedEdges); + MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_UPDATE_METRIC).inc(mergedEdges.size()); mergedEdges.forEach(graphService::upsertEdge); } } @@ -437,7 +446,7 @@ private void deleteGraphData( final HashMap> urnToRelationshipTypesBeingAdded = edgeAndRelationTypes.getSecond(); - if (urnToRelationshipTypesBeingAdded.size() > 0) { + if (!urnToRelationshipTypesBeingAdded.isEmpty()) { for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { graphService.removeEdgesFromNode( opContext, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index 3795fd19316b1..326447b3d9d8c 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -26,6 +26,7 @@ import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.timeseries.transformer.TimeseriesAspectTransformer; +import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataChangeLog; import com.linkedin.mxe.SystemMetadata; import com.linkedin.structured.StructuredPropertyDefinition; @@ -61,6 +62,9 @@ public class UpdateIndicesService implements SearchIndicesService { @Getter private final boolean structuredPropertiesWriteEnabled; + private static final String DOCUMENT_TRANSFORM_FAILED_METRIC = "document_transform_failed"; + private static final String SEARCH_DIFF_MODE_SKIPPED_METRIC = "search_diff_no_changes_detected"; + private static final Set UPDATE_CHANGE_TYPES = ImmutableSet.of( ChangeType.CREATE, @@ -283,11 +287,13 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev event.getAuditStamp())); } catch (Exception e) { log.error( - "Error in getting documents from aspect: {} for aspect {}", e, aspectSpec.getName()); + "Error in getting documents for urn: {} from aspect: {}", urn, aspectSpec.getName(), e); + MetricUtils.counter(this.getClass(), DOCUMENT_TRANSFORM_FAILED_METRIC).inc(); return; } - if (!searchDocument.isPresent()) { + if (searchDocument.isEmpty()) { + log.info("Search document for urn: {} aspect: {} was empty", urn, aspect); return; } @@ -304,15 +310,22 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev opContext, urn, previousAspect, aspectSpec, false); } catch (Exception e) { log.error( - "Error in getting documents from previous aspect state: {} for aspect {}, continuing without diffing.", - e, - aspectSpec.getName()); + "Error in getting documents from previous aspect state for urn: {} for aspect {}, continuing without diffing.", + urn, + aspectSpec.getName(), + e); + MetricUtils.counter(this.getClass(), DOCUMENT_TRANSFORM_FAILED_METRIC).inc(); } } if (previousSearchDocument.isPresent()) { if (searchDocument.get().toString().equals(previousSearchDocument.get().toString())) { // No changes to search document, skip writing no-op update + log.info( + "No changes detected for search document for urn: {} aspect: {}", + urn, + aspectSpec.getName()); + MetricUtils.counter(this.getClass(), SEARCH_DIFF_MODE_SKIPPED_METRIC).inc(); return; } } diff --git a/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml index a9a30916c38c5..474c210c4430b 100644 --- a/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml +++ b/metadata-jobs/mae-consumer-job/src/main/resources/logback.xml @@ -3,7 +3,7 @@ - %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n + %date{ISO8601} [%thread] %-5level %logger{36} urn=%X{entityUrn:-none} aspect=%X{aspectName:-none} entityType=%X{entityType:-none} changeType=%X{changeType:-none} - %msg%n scanned from multiple locations diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java index 70b452722abc7..ff929603514c5 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java @@ -1,18 +1,27 @@ package com.linkedin.metadata.kafka; +import static com.linkedin.metadata.Constants.MDC_ASPECT_NAME; +import static com.linkedin.metadata.Constants.MDC_CHANGE_TYPE; +import static com.linkedin.metadata.Constants.MDC_ENTITY_TYPE; +import static com.linkedin.metadata.Constants.MDC_ENTITY_URN; + import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import com.linkedin.common.urn.Urn; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.EventUtils; import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; import com.linkedin.metadata.utils.metrics.MetricUtils; import com.linkedin.mxe.MetadataChangeLog; import io.datahubproject.metadata.context.OperationContext; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.slf4j.MDC; @Slf4j public class MCLKafkaListener { @@ -66,13 +75,23 @@ public void consume(final ConsumerRecord consumerRecord) return; } + Urn entityUrn = event.getEntityUrn(); + String aspectName = event.hasAspectName() ? event.getAspectName() : null; + String entityType = event.hasEntityType() ? event.getEntityType() : null; + ChangeType changeType = event.hasChangeType() ? event.getChangeType() : null; + MDC.put(MDC_ENTITY_URN, Optional.ofNullable(entityUrn).map(Urn::toString).orElse("")); + MDC.put(MDC_ASPECT_NAME, aspectName); + MDC.put(MDC_ENTITY_TYPE, entityType); + MDC.put( + MDC_CHANGE_TYPE, Optional.ofNullable(changeType).map(ChangeType::toString).orElse("")); + log.info( "Invoking MCL hooks for consumer: {} urn: {}, aspect name: {}, entity type: {}, change type: {}", consumerGroupId, - event.getEntityUrn(), - event.hasAspectName() ? event.getAspectName() : null, - event.hasEntityType() ? event.getEntityType() : null, - event.hasChangeType() ? event.getChangeType() : null); + entityUrn, + aspectName, + entityType, + changeType); // Here - plug in additional "custom processor hooks" for (MetadataChangeLogHook hook : this.hooks) { @@ -98,6 +117,8 @@ public void consume(final ConsumerRecord consumerRecord) "Successfully completed MCL hooks for consumer: {} urn: {}", consumerGroupId, event.getEntityUrn()); + } finally { + MDC.clear(); } } } diff --git a/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml b/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml index f94088735fa29..f8674dba82916 100644 --- a/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml +++ b/metadata-jobs/mce-consumer-job/src/main/resources/logback.xml @@ -3,7 +3,7 @@ - %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n + %date{ISO8601} [%thread] %-5level %logger{36} urn=%X{entityUrn:-none} aspect=%X{aspectName:-none} entityType=%X{entityType:-none} changeType=%X{changeType:-none} - %msg%n scanned from multiple locations diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java index 2f3f35697e476..22c2b4b9c0450 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java @@ -1,9 +1,16 @@ package com.linkedin.metadata.kafka; +import static com.linkedin.metadata.Constants.MDC_ASPECT_NAME; +import static com.linkedin.metadata.Constants.MDC_CHANGE_TYPE; +import static com.linkedin.metadata.Constants.MDC_ENTITY_TYPE; +import static com.linkedin.metadata.Constants.MDC_ENTITY_URN; + import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; +import com.linkedin.common.urn.Urn; import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; import com.linkedin.metadata.EventUtils; @@ -27,6 +34,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; +import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Conditional; @@ -128,6 +136,17 @@ public void consume(final ConsumerRecord consumerRecord) MetadataChangeProposal event = new MetadataChangeProposal(); try { event = EventUtils.avroToPegasusMCP(record); + + Urn entityUrn = event.getEntityUrn(); + String aspectName = event.hasAspectName() ? event.getAspectName() : null; + String entityType = event.hasEntityType() ? event.getEntityType() : null; + ChangeType changeType = event.hasChangeType() ? event.getChangeType() : null; + MDC.put(MDC_ENTITY_URN, Optional.ofNullable(entityUrn).map(Urn::toString).orElse("")); + MDC.put(MDC_ASPECT_NAME, aspectName); + MDC.put(MDC_ENTITY_TYPE, entityType); + MDC.put( + MDC_CHANGE_TYPE, Optional.ofNullable(changeType).map(ChangeType::toString).orElse("")); + log.debug("MetadataChangeProposal {}", event); // TODO: Get this from the event itself. String urn = entityClient.ingestProposal(systemOperationContext, event, false); @@ -137,6 +156,8 @@ public void consume(final ConsumerRecord consumerRecord) log.error("Message: {}", record); sendFailedMCP(event, throwable); } + } finally { + MDC.clear(); } } From 8b062eb8bd70d0d7121267fe181d5bcfd7c9a40b Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Wed, 23 Oct 2024 22:39:57 +0100 Subject: [PATCH 05/26] feat(ingest/oracle): retire deprecated cx_oracle library (#11607) Co-authored-by: Harshal Sheth --- metadata-ingestion/setup.py | 2 +- .../datahub/ingestion/source/sql/oracle.py | 19 ++++++++++++------- .../tests/unit/test_oracle_source.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 35dbff5cc2c71..12614f9ff36b2 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -449,7 +449,7 @@ # mariadb should have same dependency as mysql "mariadb": sql_common | {"pymysql>=1.0.2"}, "okta": {"okta~=1.7.0", "nest-asyncio"}, - "oracle": sql_common | {"cx_Oracle"}, + "oracle": sql_common | {"oracledb"}, "postgres": sql_common | postgres_common, "presto": sql_common | pyhive_common | trino, # presto-on-hive is an alias for hive-metastore and needs to be kept in sync diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py index dcc1340c81d7b..3823c123d06cf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py @@ -1,11 +1,13 @@ import logging import re + +# This import verifies that the dependencies are available. +import sys from collections import defaultdict from typing import Any, Dict, Iterable, List, NoReturn, Optional, Tuple, Union, cast from unittest.mock import patch -# This import verifies that the dependencies are available. -import cx_Oracle +import oracledb import pydantic import sqlalchemy.engine from pydantic.fields import Field @@ -31,6 +33,9 @@ logger = logging.getLogger(__name__) +oracledb.version = "8.3.0" +sys.modules["cx_Oracle"] = oracledb + extra_oracle_types = { make_sqlalchemy_type("SDO_GEOMETRY"), make_sqlalchemy_type("SDO_POINT_TYPE"), @@ -47,10 +52,10 @@ def _raise_err(exc: Exception) -> NoReturn: def output_type_handler(cursor, name, defaultType, size, precision, scale): """Add CLOB and BLOB support to Oracle connection.""" - if defaultType == cx_Oracle.CLOB: - return cursor.var(cx_Oracle.LONG_STRING, arraysize=cursor.arraysize) - elif defaultType == cx_Oracle.BLOB: - return cursor.var(cx_Oracle.LONG_BINARY, arraysize=cursor.arraysize) + if defaultType == oracledb.CLOB: + return cursor.var(oracledb.DB_TYPE_LONG, arraysize=cursor.arraysize) + elif defaultType == oracledb.BLOB: + return cursor.var(oracledb.DB_TYPE_LONG_RAW, arraysize=cursor.arraysize) def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): @@ -60,7 +65,7 @@ def before_cursor_execute(conn, cursor, statement, parameters, context, executem class OracleConfig(BasicSQLAlchemyConfig): # defaults scheme: str = Field( - default="oracle+cx_oracle", + default="oracle+oracledb", description="Will be set automatically to default value.", ) service_name: Optional[str] = Field( diff --git a/metadata-ingestion/tests/unit/test_oracle_source.py b/metadata-ingestion/tests/unit/test_oracle_source.py index 18d4e85296e36..ce7952cf73a8a 100644 --- a/metadata-ingestion/tests/unit/test_oracle_source.py +++ b/metadata-ingestion/tests/unit/test_oracle_source.py @@ -21,7 +21,7 @@ def test_oracle_config(): ) assert ( config.get_sql_alchemy_url() - == "oracle+cx_oracle://user:password@host:1521/?service_name=svc01" + == "oracle+oracledb://user:password@host:1521/?service_name=svc01" ) with pytest.raises(ValueError): From 6787db7124861b2fdd35d073bf046bd5786425c0 Mon Sep 17 00:00:00 2001 From: Semion Sidorenko Date: Wed, 23 Oct 2024 23:42:27 +0200 Subject: [PATCH 06/26] fix(ingest/superset): Don't set schema/db for druid (#11682) --- .../src/datahub/ingestion/source/superset.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 4e40407fba908..5ce33da5c55fa 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -82,6 +82,8 @@ "box_plot": ChartTypeClass.BAR, } +platform_without_databases = ["druid"] + class SupersetConfig( StatefulIngestionConfigBase, EnvConfigMixin, PlatformInstanceConfigMixin @@ -252,17 +254,30 @@ def get_datasource_urn_from_id(self, datasource_id): dataset_response = self.session.get( f"{self.config.connect_uri}/api/v1/dataset/{datasource_id}" ).json() + schema_name = dataset_response.get("result", {}).get("schema") table_name = dataset_response.get("result", {}).get("table_name") database_id = dataset_response.get("result", {}).get("database", {}).get("id") + platform = self.get_platform_from_database_id(database_id) + database_name = ( dataset_response.get("result", {}).get("database", {}).get("database_name") ) database_name = self.config.database_alias.get(database_name, database_name) + # Druid do not have a database concept and has a limited schema concept, but they are nonetheless reported + # from superset. There is only one database per platform instance, and one schema named druid, so it would be + # redundant to systemically store them both in the URN. + if platform in platform_without_databases: + database_name = None + + if platform == "druid" and schema_name == "druid": + # Follow DataHub's druid source convention. + schema_name = None + if database_id and table_name: return make_dataset_urn( - platform=self.get_platform_from_database_id(database_id), + platform=platform, name=".".join( name for name in [database_name, schema_name, table_name] if name ), From 707a02c0365ac9dcda1b63bcd28a31ac75385df8 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 23 Oct 2024 17:22:53 -0500 Subject: [PATCH 07/26] fix(ci): remove pip cache (#11702) --- .github/workflows/build-and-test.yml | 6 +++++- .github/workflows/documentation.yml | 6 +++++- .github/workflows/metadata-ingestion.yml | 5 ----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7d6df18795219..532ba1102ed57 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -70,7 +70,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.10" - cache: pip + - uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} - name: Set up JDK 17 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index dbeff7a9c0f9f..e6044badb1b41 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -44,7 +44,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.10" - cache: pip + - uses: actions/cache@v4 + with: + path: | + ~/.cache/uv + key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} - name: Install Python dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Build Docs diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 92dcb3d8ac289..c0eafe891fb0a 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -62,16 +62,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: "pip" - uses: actions/cache@v4 with: path: | ~/.cache/uv key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Install package From f6760436b7e32909904f04e53d9cdff23f740fc0 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 24 Oct 2024 16:06:00 +0200 Subject: [PATCH 08/26] feat(ingestion/dagster): Dagster assetless ingestion (#11262) Co-authored-by: shubhamjagtap639 --- .../client/dagster_generator.py | 179 ++++- .../sensors/datahub_sensors.py | 282 +++++--- .../dagster-plugin/tests/unit/conftest.py | 21 + .../golden_test_emit_metadata_mcps.json | 658 ++++++++++++++++++ .../dagster-plugin/tests/unit/test_dagster.py | 295 +++----- .../tests/unit/utils/__init__.py | 0 .../dagster-plugin/tests/unit/utils/utils.py | 30 + metadata-ingestion/developing.md | 9 + 8 files changed, 1161 insertions(+), 313 deletions(-) create mode 100644 metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py create mode 100644 metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json create mode 100644 metadata-ingestion-modules/dagster-plugin/tests/unit/utils/__init__.py create mode 100644 metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py index df123b127e040..2fdd0a41edf6c 100644 --- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py +++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/client/dagster_generator.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from datetime import datetime, timezone from logging import Logger from typing import Any, Callable, Dict, List, NamedTuple, Optional, Sequence, Set from urllib.parse import urlsplit @@ -12,6 +13,7 @@ TableSchemaMetadataValue, ) from dagster._core.execution.stats import RunStepKeyStatsSnapshot, StepEventStatus +from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint try: from dagster._core.snap import JobSnapshot # type: ignore[attr-defined] @@ -32,13 +34,22 @@ make_data_platform_urn, make_dataplatform_instance_urn, make_tag_urn, + make_ts_millis, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph from datahub.metadata._schema_classes import ( + AuditStampClass, BrowsePathEntryClass, BrowsePathsV2Class, GlobalTagsClass, + QueryLanguageClass, + QueryPropertiesClass, + QuerySourceClass, + QueryStatementClass, + QuerySubjectClass, + QuerySubjectsClass, TagAssociationClass, ) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( @@ -63,6 +74,7 @@ SubTypesClass, UpstreamClass, ) +from datahub.metadata.urns import CorpUserUrn from datahub.specific.dataset import DatasetPatchBuilder from datahub.utilities.urns._urn_base import Urn from datahub.utilities.urns.data_flow_urn import DataFlowUrn @@ -73,6 +85,8 @@ DAGSTER_PLATFORM = "dagster" +_DEFAULT_USER_URN = CorpUserUrn("_ingestion") + class Constant: """ @@ -212,6 +226,16 @@ class DatahubDagsterSourceConfig(DatasetSourceConfigMixin): description="Whether to materialize asset dependency in DataHub. It emits a datasetKey for each dependencies. Default is False.", ) + emit_queries: Optional[bool] = pydantic.Field( + default=False, + description="Whether to emit queries aspects. Default is False.", + ) + + emit_assets: Optional[bool] = pydantic.Field( + default=True, + description="Whether to emit assets aspects. Default is True.", + ) + debug_mode: Optional[bool] = pydantic.Field( default=False, description="Whether to enable debug mode", @@ -243,9 +267,10 @@ def job_url_generator(dagster_url: str, dagster_environment: DagsterEnvironment) return base_url -class DagsterGenerator: - asset_group_name_cache: Dict[str, str] = {} +DATAHUB_ASSET_GROUP_NAME_CACHE: Dict[str, str] = {} + +class DagsterGenerator: def __init__( self, logger: Logger, @@ -298,17 +323,16 @@ def update_asset_group_name_cache( if asset_def: for key, group_name in asset_def.group_names_by_key.items(): asset_urn = self.dataset_urn_from_asset(key.path) - DagsterGenerator.asset_group_name_cache[ - asset_urn.urn() - ] = group_name + DATAHUB_ASSET_GROUP_NAME_CACHE[asset_urn.urn()] = group_name if self.config.debug_mode: self.logger.debug( f"Asset group name cache updated: {asset_urn.urn()} -> {group_name}" ) if self.config.debug_mode: self.logger.debug( - f"Asset group name cache: {DagsterGenerator.asset_group_name_cache}" + f"Asset group name cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}" ) + self.logger.info(f"Asset group name cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}") def path_metadata_resolver(self, value: PathMetadataValue) -> Optional[DatasetUrn]: """ @@ -347,6 +371,8 @@ def generate_dataflow( job_snapshot: JobSnapshot, env: str, platform_instance: Optional[str] = None, + remove_double_underscores: bool = True, + add_asset_group_tag: bool = True, ) -> DataFlow: """ Generates a Dataflow object from an Dagster Job Snapshot @@ -358,17 +384,34 @@ def generate_dataflow( if self.dagster_environment.is_cloud: id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{job_snapshot.name}" else: - id = f"{self.dagster_environment.module}/{job_snapshot.name}" + module_name = ( + self.dagster_environment.module + if self.dagster_environment.module + else self.dagster_environment.branch + ) + id = f"{module_name}/{job_snapshot.name}" + + flow_name = job_snapshot.name + if remove_double_underscores and flow_name.split("__"): + flow_name = flow_name.split("__")[-1] dataflow = DataFlow( orchestrator=Constant.ORCHESTRATOR, id=id, env=env, - name=job_snapshot.name, + name=flow_name, platform_instance=platform_instance, ) + dataflow.description = job_snapshot.description dataflow.tags = set(job_snapshot.tags.keys()) + if add_asset_group_tag: + asset_group = self.get_asset_group_from_op_name( + job_snapshot.name.split("__") + ) + if asset_group: + dataflow.tags.add(f"asset_group:{asset_group}") + if self.config.dagster_url: dataflow.url = f"{job_url_generator(dagster_url=self.config.dagster_url, dagster_environment=self.dagster_environment)}/jobs/{job_snapshot.name}" flow_property_bag: Dict[str, str] = {} @@ -386,6 +429,8 @@ def generate_datajob( input_datasets: Dict[str, Set[DatasetUrn]], output_datasets: Dict[str, Set[DatasetUrn]], platform_instance: Optional[str] = None, + remove_double_underscores: bool = True, + add_asset_group_tag: bool = True, ) -> DataJob: """ Generates a Datajob object from an Dagster op snapshot @@ -403,8 +448,13 @@ def generate_datajob( flow_id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{job_snapshot.name}" job_id = f"{self.dagster_environment.branch}/{self.dagster_environment.module}/{op_def_snap.name}" else: - flow_id = f"{self.dagster_environment.module}/{job_snapshot.name}" - job_id = f"{self.dagster_environment.module}/{op_def_snap.name}" + module_name = ( + self.dagster_environment.module + if self.dagster_environment.module + else self.dagster_environment.branch + ) + flow_id = f"{module_name}/{job_snapshot.name}" + job_id = f"{module_name}/{op_def_snap.name}" dataflow_urn = DataFlowUrn.create_from_ids( orchestrator=Constant.ORCHESTRATOR, @@ -412,10 +462,15 @@ def generate_datajob( env=env, platform_instance=platform_instance, ) + + job_name = op_def_snap.name + if remove_double_underscores and job_name.split("__"): + job_name = job_name.split("__")[-1] + datajob = DataJob( id=job_id, flow_urn=dataflow_urn, - name=op_def_snap.name, + name=job_name, ) if self.config.dagster_url: @@ -424,6 +479,13 @@ def generate_datajob( datajob.description = op_def_snap.description datajob.tags = set(op_def_snap.tags.keys()) + if add_asset_group_tag: + asset_group = self.get_asset_group_from_op_name( + op_def_snap.name.split("__") + ) + if asset_group: + datajob.tags.add(f"asset_group:{asset_group}") + inlets: Set[DatasetUrn] = set() # Add upstream dependencies for this op for upstream_op_name in step_deps[op_def_snap.name]: @@ -667,9 +729,9 @@ def generate_asset_group_tag( if not target_urn: target_urn = asset_urn self.logger.info( - f"Getting {asset_urn.urn()} from Asset Cache: {DagsterGenerator.asset_group_name_cache}" + f"Getting {asset_urn.urn()} from Asset Cache: {DATAHUB_ASSET_GROUP_NAME_CACHE}" ) - group_name = DagsterGenerator.asset_group_name_cache.get(asset_urn.urn()) + group_name = DATAHUB_ASSET_GROUP_NAME_CACHE.get(asset_urn.urn()) if group_name: current_tags: Optional[GlobalTagsClass] = graph.get_aspect( entity_urn=target_urn.urn(), @@ -698,6 +760,20 @@ def generate_asset_group_tag( return None + def _gen_entity_browsepath_aspect( + self, + entity_urn: str, + paths: List[str], + ) -> MetadataWorkUnit: + entries = [BrowsePathEntryClass(id=path) for path in paths] + if self.config.platform_instance: + urn = make_dataplatform_instance_urn("asset", self.config.platform_instance) + entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries + return MetadataChangeProposalWrapper( + entityUrn=entity_urn, + aspect=BrowsePathsV2Class(entries), + ).as_workunit() + def emit_asset( self, graph: DataHubGraph, @@ -771,7 +847,8 @@ def emit_asset( origin=DatasetUrn.create_from_string(downstream).env, ), ) - self.logger.info(f"mcp: {mcp}") + if self.config.debug_mode: + self.logger.info(f"mcp: {mcp}") graph.emit_mcp(mcp) patch_builder = DatasetPatchBuilder(downstream) @@ -807,20 +884,38 @@ def emit_asset( for patch_event in patch_builder.build(): graph.emit_mcp(patch_event) - self.logger.info(f"asset_key: {asset_key}") + if self.config.debug_mode: + self.logger.info(f"asset_key: {asset_key}") self.generate_browse_path(asset_key=asset_key, urn=dataset_urn, graph=graph) return dataset_urn + def get_asset_group_from_op_name(self, asset_key: Sequence[str]) -> Optional[str]: + """ + Get asset group name from op name + """ + asset_urn = self.dataset_urn_from_asset(asset_key).urn() + asset_group_name = DATAHUB_ASSET_GROUP_NAME_CACHE.get(asset_urn) + if asset_group_name: + self.logger.info( + f"asset_key: {asset_key}, urn: {asset_urn}, asset_group_name: {asset_group_name}" + ) + return asset_group_name + else: + self.logger.info( + f"asset_key: {asset_key}, urn: {asset_urn} not in {DATAHUB_ASSET_GROUP_NAME_CACHE}, asset_group_name: None" + ) + + return None + def generate_browse_path( self, asset_key: Sequence[str], urn: Urn, graph: DataHubGraph ) -> None: """ Generate browse path from asset key """ - asset_group_name = DagsterGenerator.asset_group_name_cache.get( - self.dataset_urn_from_asset(asset_key).urn() - ) browsePaths: List[BrowsePathEntryClass] = [] + + asset_group_name = self.get_asset_group_from_op_name(asset_key) if asset_group_name: browsePaths.append(BrowsePathEntryClass(asset_group_name)) @@ -834,3 +929,51 @@ def generate_browse_path( ), ) graph.emit_mcp(mcp) + + def gen_query_aspect( + self, + graph: DataHubGraph, + platform: str, + query_subject_urns: List[str], + query: str, + job_urn: Optional[str] = None, + ) -> None: + """ + Generate query aspect for lineage + """ + query_id = get_query_fingerprint(query, platform) + + aspects = [ + QueryPropertiesClass( + statement=QueryStatementClass( + value=query, + language=QueryLanguageClass.SQL, + ), + source=QuerySourceClass.SYSTEM, + origin=job_urn if job_urn else None, + created=AuditStampClass( + make_ts_millis(datetime.now(tz=timezone.utc)), + actor=_DEFAULT_USER_URN.urn(), + ), + lastModified=AuditStampClass( + make_ts_millis(datetime.now(tz=timezone.utc)), + actor=_DEFAULT_USER_URN.urn(), + ), + ), + QuerySubjectsClass( + subjects=[QuerySubjectClass(entity=urn) for urn in query_subject_urns] + ), + DataPlatformInstanceClass( + platform=make_data_platform_urn(platform), + ), + SubTypesClass( + typeNames=["Query"], + ), + ] + + mcps = MetadataChangeProposalWrapper.construct_many( + entityUrn=f"urn:li:query:dagster_{query_id}", + aspects=aspects, + ) + for mcp in mcps: + graph.emit_mcp(mcp) diff --git a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py index ebb2c82d952b1..f6b0629b7ca7b 100644 --- a/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py +++ b/metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/sensors/datahub_sensors.py @@ -46,8 +46,10 @@ create_lineage_sql_parsed_result, ) from datahub.utilities.urns.dataset_urn import DatasetUrn +from datahub.utilities.urns.error import InvalidUrnError from datahub_dagster_plugin.client.dagster_generator import ( + DATAHUB_ASSET_GROUP_NAME_CACHE, Constant, DagsterEnvironment, DagsterGenerator, @@ -268,7 +270,6 @@ def get_dagster_environment( module = code_pointer.module else: context.log.error("Unable to get Module") - return None dagster_environment = DagsterEnvironment( is_cloud=os.getenv("DAGSTER_CLOUD_IS_BRANCH_DEPLOYMENT", None) is not None, @@ -326,100 +327,188 @@ def process_asset_materialization( dataset_inputs: Dict[str, Set[DatasetUrn]], dataset_outputs: Dict[str, Set[DatasetUrn]], ) -> None: - if ( - log.dagster_event - and log.dagster_event.event_type == DagsterEventType.ASSET_MATERIALIZATION - ): - assert log.step_key + if not self._is_valid_asset_materialization(log): + return - materialization = log.asset_materialization - if not materialization: - return + asset_materialization = log.asset_materialization + if asset_materialization is None: + return - properties = { - key: str(value) for (key, value) in materialization.metadata.items() - } - asset_key = materialization.asset_key.path + asset_key = asset_materialization.asset_key.path + asset_downstream_urn = self._get_asset_downstream_urn( + log, context, dagster_generator, list(asset_key) + ) - asset_downstream_urn: Optional[DatasetUrn] = None - # If DataHub Urn is set then we prefer that as downstream urn - if materialization.metadata.get("datahub_urn") and isinstance( - materialization.metadata.get("datahub_urn"), TextMetadataValue - ): - try: - asset_downstream_urn = DatasetUrn.from_string( - str(materialization.metadata["datahub_urn"].text) - ) - context.log.info( - f"asset_downstream_urn from metadata datahub_urn: {asset_downstream_urn}" - ) - except Exception as e: - context.log.error(f"Error in parsing datahub_urn: {e}") + if not asset_downstream_urn: + return + + properties = { + key: str(value) for (key, value) in asset_materialization.metadata.items() + } + upstreams, downstreams = self._process_lineage( + context=context, + dagster_generator=dagster_generator, + log=log, + asset_downstream_urn=asset_downstream_urn, + ) + + self._emit_or_connect_asset( + context, + dagster_generator, + log, + list(asset_key), + properties, + upstreams, + downstreams, + dataset_inputs, + dataset_outputs, + ) - if not asset_downstream_urn: - asset_downstream_urn = ( - dagster_generator.asset_keys_to_dataset_urn_converter(asset_key) + def _is_valid_asset_materialization(self, log: EventLogEntry) -> bool: + return ( + log.dagster_event is not None + and log.dagster_event.event_type == DagsterEventType.ASSET_MATERIALIZATION + and log.step_key is not None + and log.asset_materialization is not None + ) + + def _get_asset_downstream_urn( + self, + log: EventLogEntry, + context: RunStatusSensorContext, + dagster_generator: DagsterGenerator, + asset_key: List[str], + ) -> Optional[DatasetUrn]: + materialization = log.asset_materialization + if materialization is None: + return None + + asset_downstream_urn: Optional[DatasetUrn] = None + + if materialization.metadata.get("datahub_urn") and isinstance( + materialization.metadata.get("datahub_urn"), TextMetadataValue + ): + try: + asset_downstream_urn = DatasetUrn.from_string( + str(materialization.metadata["datahub_urn"].text) ) context.log.info( - f"asset_downstream_urn from asset keys: {asset_downstream_urn}" + f"asset_downstream_urn from metadata datahub_urn: {asset_downstream_urn}" ) + except Exception as e: + context.log.error(f"Error in parsing datahub_urn: {e}") + + if not asset_downstream_urn: + asset_downstream_urn = ( + dagster_generator.asset_keys_to_dataset_urn_converter(asset_key) + ) + context.log.info( + f"asset_downstream_urn from asset keys: {asset_downstream_urn}" + ) - if asset_downstream_urn: - context.log.info(f"asset_downstream_urn: {asset_downstream_urn}") + return asset_downstream_urn - downstreams = {asset_downstream_urn.urn()} - context.log.info(f"downstreams: {downstreams}") - upstreams: Set[str] = set() - if self.config.enable_asset_query_metadata_parsing: - try: - if ( - materialization - and materialization.metadata - and materialization.metadata.get("Query") - and isinstance( - materialization.metadata.get("Query"), TextMetadataValue - ) - ): - query_metadata = materialization.metadata.get("Query") - assert query_metadata - lineage = self.parse_sql( - context=context, - sql_query=str(query_metadata.text), - env=asset_downstream_urn.env, - platform=asset_downstream_urn.platform.replace( - "urn:li:dataPlatform:", "" - ), + def _process_lineage( + self, + context: RunStatusSensorContext, + dagster_generator: DagsterGenerator, + log: EventLogEntry, + asset_downstream_urn: DatasetUrn, + ) -> Tuple[Set[str], Set[str]]: + downstreams = {asset_downstream_urn.urn()} + upstreams: Set[str] = set() + + if ( + log.asset_materialization + and self.config.enable_asset_query_metadata_parsing + ): + try: + query_metadata = log.asset_materialization.metadata.get("Query") + if isinstance(query_metadata, TextMetadataValue): + lineage = self.parse_sql( + context=context, + sql_query=str(query_metadata.text), + env=asset_downstream_urn.env, + platform=asset_downstream_urn.platform.replace( + "urn:li:dataPlatform:", "" + ), + ) + if lineage and lineage.downstreams: + if self.config.emit_queries: + dagster_generator.gen_query_aspect( + graph=self.graph, + platform=asset_downstream_urn.platform, + query_subject_urns=lineage.upstreams + + lineage.downstreams, + query=str(query_metadata.text), ) - # To make sure we don't process select queries check if downstream is present - if lineage and lineage.downstreams: - downstreams = downstreams.union( - set(lineage.downstreams) - ) - upstreams = upstreams.union(set(lineage.upstreams)) - context.log.info( - f"Upstreams: {upstreams} Downstreams: {downstreams}" - ) - else: - context.log.info( - f"Lineage not found for {query_metadata.text}" - ) - else: - context.log.info("Query not found in metadata") - except Exception as e: - context.log.info(f"Error in processing asset logs: {e}") - - # Emitting asset with upstreams and downstreams - dataset_urn = dagster_generator.emit_asset( - self.graph, - asset_key, - materialization.description, - properties, - downstreams=downstreams, - upstreams=upstreams, - materialize_dependencies=self.config.materialize_dependencies, + downstreams = downstreams.union(set(lineage.downstreams)) + upstreams = upstreams.union(set(lineage.upstreams)) + context.log.info( + f"Upstreams: {upstreams} Downstreams: {downstreams}" + ) + else: + context.log.info(f"Lineage not found for {query_metadata.text}") + else: + context.log.info("Query not found in metadata") + except Exception as e: + context.log.exception(f"Error in processing asset logs: {e}") + + return upstreams, downstreams + + def _emit_or_connect_asset( + self, + context: RunStatusSensorContext, + dagster_generator: DagsterGenerator, + log: EventLogEntry, + asset_key: List[str], + properties: Dict[str, str], + upstreams: Set[str], + downstreams: Set[str], + dataset_inputs: Dict[str, Set[DatasetUrn]], + dataset_outputs: Dict[str, Set[DatasetUrn]], + ) -> None: + if self.config.emit_assets: + context.log.info("Emitting asset metadata...") + dataset_urn = dagster_generator.emit_asset( + self.graph, + asset_key, + log.asset_materialization.description + if log.asset_materialization + else None, + properties, + downstreams=downstreams, + upstreams=upstreams, + materialize_dependencies=self.config.materialize_dependencies, + ) + if log.step_key: + dataset_outputs[log.step_key].add(dataset_urn) + else: + context.log.info( + "Not emitting assets but connecting materialized dataset to DataJobs" + ) + if log.step_key: + dataset_outputs[log.step_key] = dataset_outputs[log.step_key].union( + [DatasetUrn.from_string(d) for d in downstreams] ) - dataset_outputs[log.step_key].add(dataset_urn) + dataset_upstreams: List[DatasetUrn] = [] + + for u in upstreams: + try: + dataset_upstreams.append(DatasetUrn.from_string(u)) + except InvalidUrnError as e: + context.log.error( + f"Error in parsing upstream dataset urn: {e}", exc_info=True + ) + continue + + dataset_inputs[log.step_key] = dataset_inputs[log.step_key].union( + dataset_upstreams + ) + context.log.info( + f"Dataset Inputs: {dataset_inputs[log.step_key]} Dataset Outputs: {dataset_outputs[log.step_key]}" + ) def process_asset_observation( self, @@ -593,7 +682,9 @@ def _emit_asset_metadata( dagster_environment=dagster_environment, ) - context.log.info("Emitting asset metadata...") + context.log.info( + f"Updating asset group name cache... {DATAHUB_ASSET_GROUP_NAME_CACHE}" + ) dagster_generator.update_asset_group_name_cache(context) return SkipReason("Asset metadata processed") @@ -681,10 +772,21 @@ def _emit_metadata( platform_instance=self.config.platform_instance, ) + if ( + dataflow.name + and dataflow.name.startswith("__ASSET_JOB") + and dataflow.name.split("__") + ): + dagster_generator.generate_browse_path( + dataflow.name.split("__"), urn=dataflow.urn, graph=self.graph + ) + dataflow.name = dataflow.name.split("__")[-1] + dataflow.emit(self.graph) if self.config.debug_mode: for mcp in dataflow.generate_mcp(): - context.log.debug(f"Emitted MCP: {mcp}") + if self.config.debug_mode: + context.log.debug(f"Emitted MCP: {mcp}") # Emit dagster job run which get mapped with datahub data process instance entity dagster_generator.emit_job_run( @@ -721,6 +823,16 @@ def _emit_metadata( input_datasets=dataset_inputs, ) + if ( + datajob.name + and datajob.name.startswith("__ASSET_JOB") + and datajob.name.split("__") + ): + dagster_generator.generate_browse_path( + datajob.name.split("__"), urn=datajob.urn, graph=self.graph + ) + datajob.name = datajob.name.split("__")[-1] + datajob.emit(self.graph) if self.config.debug_mode: diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py new file mode 100644 index 0000000000000..5b9442d048672 --- /dev/null +++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/conftest.py @@ -0,0 +1,21 @@ +import pathlib +import site +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from _pytest.config import Parser + + +def pytest_addoption(parser: "Parser") -> None: + parser.addoption( + "--update-golden-files", + action="store_true", + default=False, + ) + + +# See https://coverage.readthedocs.io/en/latest/subprocess.html#configuring-python-for-sub-process-measurement +coverage_startup_code = "import coverage; coverage.process_startup()" +site_packages_dir = pathlib.Path(site.getsitepackages()[0]) +pth_file_path = site_packages_dir / "datahub_coverage_startup.pth" +pth_file_path.write_text(coverage_startup_code) diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json b/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json new file mode 100644 index 0000000000000..afd67422d1564 --- /dev/null +++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/golden/golden_test_emit_metadata_mcps.json @@ -0,0 +1,658 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": {}, + "name": "etl", + "env": "PROD" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": {}, + "name": "etl", + "env": "PROD" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "job_snapshot_id": "28d03bf7b3138bea153a21f70f29e7c3ffa6ab0a", + "execution_plan_snapshot_id": "0f504b218cd28750ffa8d90a40e9647acba72021", + "has_repository_load_data": "False", + "tags": "{}", + "steps_succeeded": "2", + "steps_failed": "0", + "materializations": "0", + "expectations": "0", + "start_time": "1720681200.0", + "end_time": "1720681200.0" + }, + "name": "12345678123456781234567812345678", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1720681200000, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {'datahub.outputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)'])}, 'is_dynamic': False}" + }, + "name": "extract", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataFlow:(dagster,prod/etl,PROD)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:0fa780ff92120ac3bf49d0bc070bbda2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "dagster" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {'datahub.outputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)'])}, 'is_dynamic': False}" + }, + "name": "extract", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "input.data": "{'name': 'data', 'dagster_type_key': 'Any', 'description': None, 'metadata': {'datahub.inputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)'])}}", + "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {}, 'is_dynamic': False}" + }, + "name": "transform", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake", + "name": "tableA", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "step_key": "extract", + "attempts": "1", + "start_time": "1720681200.0", + "end_time": "1720681200.0" + }, + "name": "12345678123456781234567812345678.prod/extract", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1720681200000, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/extract)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:8e047f916b20c89e1874518890d6ff7e", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "dagster" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "input.data": "{'name': 'data', 'dagster_type_key': 'Any', 'description': None, 'metadata': {'datahub.inputs': JsonMetadataValue(data=['urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)'])}}", + "output_result": "{'name': 'result', 'dagster_type_key': 'Any', 'description': None, 'is_required': True, 'metadata': {}, 'is_dynamic': False}" + }, + "name": "transform", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [], + "fineGrainedLineages": [] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake", + "name": "tableA", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:dagster" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "step_key": "transform", + "attempts": "1", + "start_time": "1720681200.0", + "end_time": "1720681200.0" + }, + "name": "12345678123456781234567812345678.prod/transform", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1720681200000, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake", + "name": "tableA", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(dagster,prod/etl,PROD),prod/transform)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)" + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:6d553b12b2e5bf4cff8dc527e12e7116", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1720681200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "dagster" + } + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py index ac46cfd86fbb9..c951b959f85d4 100644 --- a/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py +++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/test_dagster.py @@ -1,6 +1,11 @@ +import json +import pathlib +import tempfile +import uuid +from typing import Dict, List, Mapping, Sequence, Set from unittest.mock import Mock, patch -import pytest +import dagster._core.utils from dagster import ( DagsterInstance, In, @@ -11,12 +16,16 @@ job, op, ) -from datahub.api.entities.dataprocess.dataprocess_instance import ( - DataProcessInstanceKey, - InstanceRunResult, +from dagster._core.definitions.job_definition import JobDefinition +from dagster._core.definitions.repository_definition import ( + RepositoryData, + RepositoryDefinition, ) -from datahub.configuration.source_common import DEFAULT_ENV +from dagster._core.definitions.resource_definition import ResourceDefinition +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.graph.client import DatahubClientConfig +from freezegun import freeze_time +from utils.utils import PytestConfig, check_golden_file from datahub_dagster_plugin.client.dagster_generator import DatahubDagsterSourceConfig from datahub_dagster_plugin.sensors.datahub_sensors import ( @@ -24,12 +33,49 @@ make_datahub_sensor, ) +FROZEN_TIME = "2024-07-11 07:00:00" -@patch("datahub.ingestion.graph.client.DataHubGraph", autospec=True) -@pytest.mark.skip(reason="disabling this test unti it will use proper golden files") +call_num = 0 + + +def make_new_run_id_mock() -> str: + global call_num + call_num += 1 + return f"test_run_id_{call_num}" + + +dagster._core.utils.make_new_run_id = make_new_run_id_mock + + +@patch("datahub_dagster_plugin.sensors.datahub_sensors.DataHubGraph", autospec=True) def test_datahub_sensor(mock_emit): instance = DagsterInstance.ephemeral() - context = build_sensor_context(instance=instance) + + class DummyRepositoryData(RepositoryData): + def __init__(self): + self.sensors = [] + + def get_all_jobs(self) -> Sequence["JobDefinition"]: + return [] + + def get_top_level_resources(self) -> Mapping[str, "ResourceDefinition"]: + """Return all top-level resources in the repository as a list, + such as those provided to the Definitions constructor. + + Returns: + List[ResourceDefinition]: All top-level resources in the repository. + """ + return {} + + def get_env_vars_by_top_level_resource(self) -> Mapping[str, Set[str]]: + return {} + + repository_defintion = RepositoryDefinition( + name="testRepository", repository_data=DummyRepositoryData() + ) + context = build_sensor_context( + instance=instance, repository_def=repository_defintion + ) mock_emit.return_value = Mock() config = DatahubDagsterSourceConfig( @@ -44,9 +90,13 @@ def test_datahub_sensor(mock_emit): assert isinstance(skip_reason, SkipReason) -@patch("datahub_dagster_plugin.sensors.datahub_sensors.DatahubClient", autospec=True) -@pytest.mark.skip(reason="disabling this test unti it will use proper golden files") -def test_emit_metadata(mock_emit): +TEST_UUIDS = ["uuid_{}".format(i) for i in range(10000)] + + +@patch.object(uuid, "uuid4", side_effect=TEST_UUIDS) +@patch("datahub_dagster_plugin.sensors.datahub_sensors.DataHubGraph", autospec=True) +@freeze_time(FROZEN_TIME) +def test_emit_metadata(mock_emit: Mock, pytestconfig: PytestConfig) -> None: mock_emitter = Mock() mock_emit.return_value = mock_emitter @@ -87,7 +137,8 @@ def etl(): transform(extract()) instance = DagsterInstance.ephemeral() - result = etl.execute_in_process(instance=instance) + test_run_id = "12345678123456781234567812345678" + result = etl.execute_in_process(instance=instance, run_id=test_run_id) # retrieve the DagsterRun dagster_run = result.dagster_run @@ -103,201 +154,25 @@ def etl(): dagster_event=dagster_event, ) - DatahubSensors()._emit_metadata(run_status_sensor_context) - - expected_dataflow_urn = ( - f"urn:li:dataFlow:(dagster,{dagster_run.job_name},{DEFAULT_ENV})" - ) - assert mock_emitter.method_calls[1][1][0].aspectName == "dataFlowInfo" - assert mock_emitter.method_calls[1][1][0].entityUrn == expected_dataflow_urn - assert mock_emitter.method_calls[2][1][0].aspectName == "ownership" - assert mock_emitter.method_calls[2][1][0].entityUrn == expected_dataflow_urn - assert mock_emitter.method_calls[3][1][0].aspectName == "globalTags" - assert mock_emitter.method_calls[3][1][0].entityUrn == expected_dataflow_urn - - dpi_id = DataProcessInstanceKey( - cluster=DEFAULT_ENV, - orchestrator="dagster", - id=dagster_run.run_id, - ).guid() - assert ( - mock_emitter.method_calls[7][1][0].aspectName == "dataProcessInstanceProperties" - ) - assert ( - mock_emitter.method_calls[7][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[8][1][0].aspectName - == "dataProcessInstanceRelationships" - ) - assert ( - mock_emitter.method_calls[8][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[9][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[9][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[10][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[10][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[10][1][0].aspect.result.type - == InstanceRunResult.SUCCESS - ) - assert mock_emitter.method_calls[11][1][0].aspectName == "dataJobInfo" - assert ( - mock_emitter.method_calls[11][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},extract)" - ) - assert mock_emitter.method_calls[12][1][0].aspectName == "dataJobInputOutput" - assert ( - mock_emitter.method_calls[12][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},extract)" - ) - assert mock_emitter.method_calls[13][1][0].aspectName == "status" - assert ( - mock_emitter.method_calls[13][1][0].entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)" - ) - assert mock_emitter.method_calls[14][1][0].aspectName == "ownership" - assert ( - mock_emitter.method_calls[14][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},extract)" - ) - assert mock_emitter.method_calls[15][1][0].aspectName == "globalTags" - assert ( - mock_emitter.method_calls[15][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},extract)" - ) - dpi_id = DataProcessInstanceKey( - cluster=DEFAULT_ENV, - orchestrator="dagster", - id=f"{dagster_run.run_id}.extract", - ).guid() - assert ( - mock_emitter.method_calls[21][1][0].aspectName - == "dataProcessInstanceProperties" - ) - assert ( - mock_emitter.method_calls[21][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[22][1][0].aspectName - == "dataProcessInstanceRelationships" - ) - assert ( - mock_emitter.method_calls[22][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert mock_emitter.method_calls[23][1][0].aspectName == "dataProcessInstanceOutput" - assert ( - mock_emitter.method_calls[23][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert mock_emitter.method_calls[24][1][0].aspectName == "status" - assert ( - mock_emitter.method_calls[24][1][0].entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableB,PROD)" - ) - assert ( - mock_emitter.method_calls[25][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[25][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[26][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[26][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[26][1][0].aspect.result.type - == InstanceRunResult.SUCCESS - ) - assert mock_emitter.method_calls[27][1][0].aspectName == "dataJobInfo" - assert ( - mock_emitter.method_calls[27][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},transform)" - ) - assert mock_emitter.method_calls[28][1][0].aspectName == "dataJobInputOutput" - assert ( - mock_emitter.method_calls[28][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},transform)" - ) - assert mock_emitter.method_calls[29][1][0].aspectName == "status" - assert ( - mock_emitter.method_calls[29][1][0].entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)" - ) - assert mock_emitter.method_calls[30][1][0].aspectName == "ownership" - assert ( - mock_emitter.method_calls[30][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},transform)" - ) - assert mock_emitter.method_calls[31][1][0].aspectName == "globalTags" - assert ( - mock_emitter.method_calls[31][1][0].entityUrn - == f"urn:li:dataJob:({expected_dataflow_urn},transform)" - ) - dpi_id = DataProcessInstanceKey( - cluster=DEFAULT_ENV, - orchestrator="dagster", - id=f"{dagster_run.run_id}.transform", - ).guid() - assert ( - mock_emitter.method_calls[37][1][0].aspectName - == "dataProcessInstanceProperties" - ) - assert ( - mock_emitter.method_calls[37][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[38][1][0].aspectName - == "dataProcessInstanceRelationships" - ) - assert ( - mock_emitter.method_calls[38][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert mock_emitter.method_calls[39][1][0].aspectName == "dataProcessInstanceInput" - assert ( - mock_emitter.method_calls[39][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert mock_emitter.method_calls[40][1][0].aspectName == "status" - assert ( - mock_emitter.method_calls[40][1][0].entityUrn - == "urn:li:dataset:(urn:li:dataPlatform:snowflake,tableA,PROD)" - ) - assert ( - mock_emitter.method_calls[41][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[41][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[42][1][0].aspectName == "dataProcessInstanceRunEvent" - ) - assert ( - mock_emitter.method_calls[42][1][0].entityUrn - == f"urn:li:dataProcessInstance:{dpi_id}" - ) - assert ( - mock_emitter.method_calls[42][1][0].aspect.result.type - == InstanceRunResult.SUCCESS - ) + with tempfile.TemporaryDirectory() as tmp_path: + DatahubSensors()._emit_metadata(run_status_sensor_context) + mcpws: List[Dict] = [] + for mock_call in mock_emitter.method_calls: + if not mock_call.args: + continue + mcpw = mock_call.args[0] + if isinstance(mcpw, MetadataChangeProposalWrapper): + mcpws.append(mcpw.to_obj(simplified_structure=True)) + + with open(f"{tmp_path}/test_emit_metadata_mcps.json", "w") as f: + json_object = json.dumps(mcpws, indent=2) + f.write(json_object) + + check_golden_file( + pytestconfig=pytestconfig, + output_path=pathlib.Path(f"{tmp_path}/test_emit_metadata_mcps.json"), + golden_path=pathlib.Path( + "tests/unit/golden/golden_test_emit_metadata_mcps.json" + ), + ignore_paths=["root[*]['systemMetadata']['created']"], + ) diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/__init__.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py new file mode 100644 index 0000000000000..79e8b758bd0c0 --- /dev/null +++ b/metadata-ingestion-modules/dagster-plugin/tests/unit/utils/utils.py @@ -0,0 +1,30 @@ +import pathlib +from typing import Sequence + +from datahub.testing.compare_metadata_json import assert_metadata_files_equal + +try: + from pytest import Config as PytestConfig # type: ignore[attr-defined] +except ImportError: + # Support for pytest 6.x. + from _pytest.config import Config as PytestConfig # type: ignore + +__all__ = ["PytestConfig"] + + +def check_golden_file( + pytestconfig: PytestConfig, + output_path: pathlib.Path, + golden_path: pathlib.Path, + ignore_paths: Sequence[str] = (), +) -> None: + update_golden = pytestconfig.getoption("--update-golden-files") + + assert_metadata_files_equal( + output_path=output_path, + golden_path=golden_path, + update_golden=update_golden, + copy_output=False, + ignore_paths=ignore_paths, + ignore_order=True, + ) diff --git a/metadata-ingestion/developing.md b/metadata-ingestion/developing.md index 19a18c5275a3b..b713997d8286f 100644 --- a/metadata-ingestion/developing.md +++ b/metadata-ingestion/developing.md @@ -89,7 +89,16 @@ cd metadata-ingestion-modules/gx-plugin source venv/bin/activate datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" ``` +### (Optional) Set up your Python environment for developing on Dagster Plugin + +From the repository root: +```shell +cd metadata-ingestion-modules/dagster-plugin +../../gradlew :metadata-ingestion-modules:dagster-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` ### Common setup issues Common issues (click to expand): From a2bce655cff97ef1c6eb27bf44cbcf1964fa3166 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Oct 2024 11:54:57 -0500 Subject: [PATCH 09/26] chore(deps): bump http-proxy-middleware from 2.0.6 to 2.0.7 in /docs-website (#11694) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs-website/yarn.lock | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 039dac5d5556b..0c4f2358d7139 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -6347,9 +6347,9 @@ http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1: debug "^4.3.4" http-proxy-middleware@^2.0.3: - version "2.0.6" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f" - integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== + version "2.0.7" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" + integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1" @@ -9708,6 +9708,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" +"react-loadable@npm:@docusaurus/react-loadable@5.5.2": + version "5.5.2" + resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" + integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== + dependencies: + "@types/react" "*" + prop-types "^15.6.2" + react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" From 4f6f69d0769b1b4a3ee632644631c056b38c3abe Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 24 Oct 2024 19:24:44 +0200 Subject: [PATCH 10/26] fet(ingest/bigquery): Add support ingesting foreign keys and primary keys for BigQuery tables (#11686) --- .../source/bigquery_v2/bigquery_config.py | 5 + .../source/bigquery_v2/bigquery_report.py | 3 +- .../source/bigquery_v2/bigquery_schema.py | 66 ++++++++++++ .../source/bigquery_v2/bigquery_schema_gen.py | 102 +++++++++++++++++- .../ingestion/source/bigquery_v2/queries.py | 24 +++++ 5 files changed, 196 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index 2a34fb82c2050..1f74fffeab7fb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -369,6 +369,11 @@ class BigQueryV2Config( description="Capture BigQuery dataset labels as DataHub tag", ) + include_table_constraints: bool = Field( + default=True, + description="Whether to ingest table constraints. If you know you don't use table constraints, you can disable it to save one extra query per dataset. In general it should be enabled", + ) + include_external_url: bool = Field( default=True, description="Whether to populate BigQuery Console url to Datasets/Tables", diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py index cc8c6482eb9ef..88c6eb1885f3b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py @@ -30,6 +30,7 @@ class BigQuerySchemaApiPerfReport(Report): num_list_tables_api_requests: int = 0 num_get_views_for_dataset_api_requests: int = 0 num_get_snapshots_for_dataset_api_requests: int = 0 + num_get_table_constraints_for_dataset_api_requests: int = 0 list_projects_timer: PerfTimer = field(default_factory=PerfTimer) list_projects_with_labels_timer: PerfTimer = field(default_factory=PerfTimer) @@ -37,6 +38,7 @@ class BigQuerySchemaApiPerfReport(Report): get_columns_for_dataset_sec: float = 0 get_tables_for_dataset_sec: float = 0 + get_table_constraints_for_dataset_sec: float = 0 list_tables_sec: float = 0 get_views_for_dataset_sec: float = 0 get_snapshots_for_dataset_sec: float = 0 @@ -155,7 +157,6 @@ class BigQueryV2Report( num_filtered_query_events: int = 0 num_usage_query_hash_collisions: int = 0 num_operational_stats_workunits_emitted: int = 0 - snapshots_scanned: int = 0 # view lineage diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index dd683559a007b..6361d5f266cb7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -42,6 +42,20 @@ class BigqueryColumn(BaseColumn): policy_tags: Optional[List[str]] = None +@dataclass +class BigqueryTableConstraint: + name: str + project_id: str + dataset_name: str + table_name: str + type: str + field_path: str + referenced_project_id: Optional[str] = None + referenced_dataset: Optional[str] = None + referenced_table_name: Optional[str] = None + referenced_column_name: Optional[str] = None + + RANGE_PARTITION_NAME: str = "RANGE" @@ -106,6 +120,7 @@ class BigqueryTable(BaseTable): partition_info: Optional[PartitionInfo] = None columns_ignore_from_profiling: List[str] = field(default_factory=list) external: bool = False + constraints: List[BigqueryTableConstraint] = field(default_factory=list) @dataclass @@ -492,6 +507,57 @@ def get_policy_tags_for_column( exc=e, ) + def get_table_constraints_for_dataset( + self, + project_id: str, + dataset_name: str, + report: BigQueryV2Report, + ) -> Optional[Dict[str, List[BigqueryTableConstraint]]]: + constraints: Dict[str, List[BigqueryTableConstraint]] = defaultdict(list) + with PerfTimer() as timer: + try: + cur = self.get_query_result( + BigqueryQuery.constraints_for_table.format( + project_id=project_id, dataset_name=dataset_name + ) + ) + except Exception as e: + report.warning( + title="Failed to retrieve table constraints for dataset", + message="Query to get table constraints for dataset failed with exception", + context=f"{project_id}.{dataset_name}", + exc=e, + ) + return None + + for constraint in cur: + constraints[constraint.table_name].append( + BigqueryTableConstraint( + name=constraint.constraint_name, + project_id=constraint.table_catalog, + dataset_name=constraint.table_schema, + table_name=constraint.table_name, + type=constraint.constraint_type, + field_path=constraint.column_name, + referenced_project_id=constraint.referenced_catalog + if constraint.constraint_type == "FOREIGN KEY" + else None, + referenced_dataset=constraint.referenced_schema + if constraint.constraint_type == "FOREIGN KEY" + else None, + referenced_table_name=constraint.referenced_table + if constraint.constraint_type == "FOREIGN KEY" + else None, + referenced_column_name=constraint.referenced_column + if constraint.constraint_type == "FOREIGN KEY" + else None, + ) + ) + self.report.num_get_table_constraints_for_dataset_api_requests += 1 + self.report.get_table_constraints_for_dataset_sec += timer.elapsed_seconds() + + return constraints + def get_columns_for_dataset( self, project_id: str, diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index f53642d1fead2..dc53e2f74959e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -2,13 +2,18 @@ import re from base64 import b32decode from collections import defaultdict +from itertools import groupby from typing import Dict, Iterable, List, Optional, Set, Type, Union, cast from google.cloud.bigquery.table import TableListItem from datahub.api.entities.platformresource.platform_resource import PlatformResource from datahub.configuration.pattern_utils import is_schema_allowed, is_tag_allowed -from datahub.emitter.mce_builder import make_tag_urn +from datahub.emitter.mce_builder import ( + make_dataset_urn_with_platform_instance, + make_schema_field_urn, + make_tag_urn, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import BigQueryDatasetKey, ContainerKey, ProjectIdKey from datahub.ingestion.api.workunit import MetadataWorkUnit @@ -39,6 +44,7 @@ BigqueryProject, BigQuerySchemaApi, BigqueryTable, + BigqueryTableConstraint, BigqueryTableSnapshot, BigqueryView, ) @@ -76,6 +82,7 @@ BooleanType, BytesType, DateType, + ForeignKeyConstraint, MySqlDDL, NullType, NumberType, @@ -470,6 +477,7 @@ def _process_schema( ) columns = None + constraints: Optional[Dict[str, List[BigqueryTableConstraint]]] = None rate_limiter: Optional[RateLimiter] = None if self.config.rate_limit: @@ -487,6 +495,10 @@ def _process_schema( report=self.report, rate_limiter=rate_limiter, ) + if self.config.include_table_constraints: + constraints = self.schema_api.get_table_constraints_for_dataset( + project_id=project_id, dataset_name=dataset_name, report=self.report + ) elif self.store_table_refs: # Need table_refs to calculate lineage and usage for table_item in self.schema_api.list_tables(dataset_name, project_id): @@ -516,6 +528,11 @@ def _process_schema( for table in db_tables[dataset_name]: table_columns = columns.get(table.name, []) if columns else [] + table_constraints = ( + constraints.get(table.name, []) if constraints else [] + ) + + table.constraints = table_constraints table_wu_generator = self._process_table( table=table, columns=table_columns, @@ -722,6 +739,58 @@ def make_tag_urn_from_label(self, key: str, value: str) -> str: else: return make_tag_urn(key) + # New method to generate ForeignKeyConstraint aspects + def gen_foreign_keys( + self, + table: BigqueryTable, + dataset_name: str, + project_id: str, + ) -> Iterable[ForeignKeyConstraint]: + table_id = f"{project_id}.{dataset_name}.{table.name}" + foreign_keys: List[BigqueryTableConstraint] = list( + filter(lambda x: x.type == "FOREIGN KEY", table.constraints) + ) + for key, group in groupby( + foreign_keys, + lambda x: f"{x.referenced_project_id}.{x.referenced_dataset}.{x.referenced_table_name}", + ): + dataset_urn = make_dataset_urn_with_platform_instance( + platform="bigquery", + name=table_id, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + + foreign_dataset = make_dataset_urn_with_platform_instance( + platform="bigquery", + name=key, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) + source_fields: List[str] = list() + referenced_fields: List[str] = list() + + for item in group: + source_field = make_schema_field_urn( + parent_urn=dataset_urn, field_path=item.field_path + ) + assert item.referenced_column_name + referenced_field = make_schema_field_urn( + parent_urn=foreign_dataset, field_path=item.referenced_column_name + ) + + source_fields.append(source_field) + referenced_fields.append(referenced_field) + + foreign_key_aspect = ForeignKeyConstraint( + name=key, + foreignFields=referenced_fields, + sourceFields=source_fields, + foreignDataset=foreign_dataset, + ) + + yield foreign_key_aspect + def gen_table_dataset_workunits( self, table: BigqueryTable, @@ -968,7 +1037,17 @@ def gen_tags_aspect_workunit( entityUrn=dataset_urn, aspect=tags ).as_workunit() - def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]: + def is_primary_key( + self, field_path: str, constraints: List[BigqueryTableConstraint] + ) -> bool: + for constraint in constraints: + if constraint.field_path == field_path and constraint.type == "PRIMARY KEY": + return True + return False + + def gen_schema_fields( + self, columns: List[BigqueryColumn], constraints: List[BigqueryTableConstraint] + ) -> List[SchemaField]: schema_fields: List[SchemaField] = [] # Below line affects HiveColumnToAvroConverter._STRUCT_TYPE_SEPARATOR in global scope @@ -1029,6 +1108,7 @@ def gen_schema_fields(self, columns: List[BigqueryColumn]) -> List[SchemaField]: self.BIGQUERY_FIELD_TYPE_MAPPINGS.get(col.data_type, NullType)() ), isPartitioningKey=col.is_partition_column, + isPartOfKey=self.is_primary_key(col.field_path, constraints), nativeDataType=col.data_type, description=col.comment, nullable=col.is_nullable, @@ -1048,6 +1128,16 @@ def gen_schema_metadata( columns: List[BigqueryColumn], dataset_name: BigqueryTableIdentifier, ) -> MetadataWorkUnit: + + foreign_keys: List[ForeignKeyConstraint] = [] + # Foreign keys only make sense for tables + if isinstance(table, BigqueryTable): + foreign_keys = list( + self.gen_foreign_keys( + table, dataset_name.dataset, dataset_name.project_id + ) + ) + schema_metadata = SchemaMetadata( schemaName=str(dataset_name), platform=self.identifiers.make_data_platform_urn(), @@ -1055,7 +1145,13 @@ def gen_schema_metadata( hash="", platformSchema=MySqlDDL(tableSchema=""), # fields=[], - fields=self.gen_schema_fields(columns), + fields=self.gen_schema_fields( + columns, + table.constraints + if (isinstance(table, BigqueryTable) and table.constraints) + else [], + ), + foreignKeys=foreign_keys if foreign_keys else None, ) if self.config.lineage_parse_view_ddl or self.config.lineage_use_sql_parser: diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py index 6155619d144c4..2ac40a48de4cc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries.py @@ -294,6 +294,30 @@ class BigqueryQuery: ORDER BY table_catalog, table_schema, table_name, ordinal_position ASC, data_type DESC""" + constraints_for_table: str = """ +select + kcu.constraint_name as constraint_name, + kcu.table_catalog as table_catalog, + kcu.table_schema as table_schema, + kcu.table_name as table_name, + kcu.column_name as column_name, + tc.constraint_type, + ccu.table_catalog as referenced_catalog, + ccu.table_schema as referenced_schema, + ccu.table_name as referenced_table, + ccu.column_name as referenced_column +from + `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE as kcu +join `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE as ccu on + kcu.constraint_catalog = ccu.constraint_catalog + and kcu.constraint_schema = ccu.constraint_schema + and kcu.constraint_name = ccu.constraint_name +join `{project_id}`.`{dataset_name}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS as tc on + tc.constraint_catalog = ccu.constraint_catalog + and tc.constraint_schema = ccu.constraint_schema + and tc.constraint_name = ccu.constraint_name +""" + BQ_FILTER_RULE_TEMPLATE_V2_LINEAGE = """ resource.type=("bigquery_project") From b89ca3f081d56022d9da3597490175b87f9bce54 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 24 Oct 2024 10:31:34 -0700 Subject: [PATCH 11/26] feat(ingest/fivetran): support overriding destination db (#11701) --- docs/how/updating-datahub.md | 2 + .../ingestion/source/fivetran/config.py | 46 +++++++++++++--- .../ingestion/source/fivetran/fivetran.py | 54 ++++++++++--------- .../source/fivetran/fivetran_query.py | 6 +-- .../integration/fivetran/test_fivetran.py | 41 +++++++++++--- 5 files changed, 107 insertions(+), 42 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 8911d282f86bb..db97f7aa81d7b 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -33,6 +33,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. +- #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance..database`. + ### Breaking Changes - #11486 - Deprecated Criterion filters using `value`. Use `values` instead. This also deprecates the ability to use comma delimited string to represent multiple values using `value`. diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py index 394015500d1c5..1e15f6b395ca5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py @@ -1,12 +1,17 @@ import dataclasses import logging +import warnings from typing import Dict, Optional import pydantic from pydantic import Field, root_validator from typing_extensions import Literal -from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.configuration.common import ( + AllowDenyPattern, + ConfigModel, + ConfigurationWarning, +) from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.ingestion.api.report import Report @@ -144,6 +149,10 @@ def report_connectors_dropped(self, model: str) -> None: class PlatformDetail(ConfigModel): + platform: Optional[str] = pydantic.Field( + default=None, + description="Override the platform type detection.", + ) platform_instance: Optional[str] = pydantic.Field( default=None, description="The instance of the platform that all assets produced by this recipe belong to", @@ -152,6 +161,11 @@ class PlatformDetail(ConfigModel): default=DEFAULT_ENV, description="The environment that all assets produced by DataHub platform ingestion source belong to", ) + database: Optional[str] = pydantic.Field( + default=None, + description="The database that all assets produced by this connector belong to. " + "For destinations, this defaults to the fivetran log config's database.", + ) class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin): @@ -172,24 +186,42 @@ class FivetranSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin default=True, description="Populates table->table column lineage.", ) - sources_to_database: Dict[str, str] = pydantic.Field( - default={}, - description="A mapping of the connector's all sources to its database. Use connector id as key.", - ) + # Configuration for stateful ingestion stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( default=None, description="Airbyte Stateful Ingestion Config." ) + # Fivetran connector all sources to platform instance mapping sources_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( default={}, - description="A mapping of the connector's all sources dataset to platform instance. Use connector id as key.", + description="A mapping from connector id to its platform/instance/env/database details.", ) # Fivetran destination to platform instance mapping destination_to_platform_instance: Dict[str, PlatformDetail] = pydantic.Field( default={}, - description="A mapping of destination dataset to platform instance. Use destination id as key.", + description="A mapping of destination id to its platform/instance/env details.", ) + + @pydantic.root_validator(pre=True) + def compat_sources_to_database(cls, values: Dict) -> Dict: + if "sources_to_database" in values: + warnings.warn( + "The sources_to_database field is deprecated, please use sources_to_platform_instance instead.", + ConfigurationWarning, + stacklevel=2, + ) + mapping = values.pop("sources_to_database") + + values.setdefault("sources_to_platform_instance", {}) + for key, value in mapping.items(): + values["sources_to_platform_instance"].setdefault(key, {}) + values["sources_to_platform_instance"][key].setdefault( + "database", value + ) + + return values + history_sync_lookback_period: int = pydantic.Field( 7, description="The number of days to look back when extracting connectors' sync history.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py index c27ec57c2e99e..907bfa3a167aa 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py @@ -81,31 +81,33 @@ def _extend_lineage(self, connector: Connector, datajob: DataJob) -> None: output_dataset_urn_list: List[DatasetUrn] = [] fine_grained_lineage: List[FineGrainedLineage] = [] - source_platform_detail: PlatformDetail = PlatformDetail() - destination_platform_detail: PlatformDetail = PlatformDetail() + # TODO: Once Fivetran exposes the database via the API, we shouldn't ask for it via config. + # Get platform details for connector source - source_platform_detail = self.config.sources_to_platform_instance.get( + source_details = self.config.sources_to_platform_instance.get( connector.connector_id, PlatformDetail() ) + if source_details.platform is None: + if connector.connector_type in KNOWN_DATA_PLATFORM_MAPPING: + source_details.platform = KNOWN_DATA_PLATFORM_MAPPING[ + connector.connector_type + ] + else: + logger.info( + f"Fivetran connector source type: {connector.connector_type} is not supported to mapped with Datahub dataset entity." + ) + source_details.platform = connector.connector_type # Get platform details for destination - destination_platform_detail = self.config.destination_to_platform_instance.get( + destination_details = self.config.destination_to_platform_instance.get( connector.destination_id, PlatformDetail() ) - - # Get database for connector source - # TODO: Once Fivetran exposes this, we shouldn't ask for it via config. - source_database: Optional[str] = self.config.sources_to_database.get( - connector.connector_id - ) - - if connector.connector_type in KNOWN_DATA_PLATFORM_MAPPING: - source_platform = KNOWN_DATA_PLATFORM_MAPPING[connector.connector_type] - else: - source_platform = connector.connector_type - logger.info( - f"Fivetran connector source type: {connector.connector_type} is not supported to mapped with Datahub dataset entity." + if destination_details.platform is None: + destination_details.platform = ( + self.config.fivetran_log_config.destination_platform ) + if destination_details.database is None: + destination_details.database = self.audit_log.fivetran_log_database if len(connector.lineage) >= MAX_TABLE_LINEAGE_PER_CONNECTOR: self.report.warning( @@ -117,22 +119,22 @@ def _extend_lineage(self, connector: Connector, datajob: DataJob) -> None: for lineage in connector.lineage: input_dataset_urn = DatasetUrn.create_from_ids( - platform_id=source_platform, + platform_id=source_details.platform, table_name=( - f"{source_database.lower()}.{lineage.source_table}" - if source_database + f"{source_details.database.lower()}.{lineage.source_table}" + if source_details.database else lineage.source_table ), - env=source_platform_detail.env, - platform_instance=source_platform_detail.platform_instance, + env=source_details.env, + platform_instance=source_details.platform_instance, ) input_dataset_urn_list.append(input_dataset_urn) output_dataset_urn = DatasetUrn.create_from_ids( - platform_id=self.config.fivetran_log_config.destination_platform, - table_name=f"{self.audit_log.fivetran_log_database.lower()}.{lineage.destination_table}", - env=destination_platform_detail.env, - platform_instance=destination_platform_detail.platform_instance, + platform_id=destination_details.platform, + table_name=f"{destination_details.database.lower()}.{lineage.destination_table}", + env=destination_details.env, + platform_instance=destination_details.platform_instance, ) output_dataset_urn_list.append(output_dataset_urn) diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py index 34dd252ec72b7..39c4d7712b4fc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran_query.py @@ -1,9 +1,9 @@ from typing import List # Safeguards to prevent fetching massive amounts of data. -MAX_TABLE_LINEAGE_PER_CONNECTOR = 100 -MAX_COLUMN_LINEAGE_PER_CONNECTOR = 3000 -MAX_JOBS_PER_CONNECTOR = 1000 +MAX_TABLE_LINEAGE_PER_CONNECTOR = 50 +MAX_COLUMN_LINEAGE_PER_CONNECTOR = 500 +MAX_JOBS_PER_CONNECTOR = 500 class FivetranLogQuery: diff --git a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py index f49f499fe43b4..2e6c2b1370d16 100644 --- a/metadata-ingestion/tests/integration/fivetran/test_fivetran.py +++ b/metadata-ingestion/tests/integration/fivetran/test_fivetran.py @@ -13,6 +13,7 @@ from datahub.ingestion.source.fivetran.config import ( BigQueryDestinationConfig, FivetranSourceConfig, + PlatformDetail, SnowflakeDestinationConfig, ) from datahub.ingestion.source.fivetran.fivetran import FivetranSource @@ -180,11 +181,9 @@ def test_fivetran_with_snowflake_dest(pytestconfig, tmp_path): "interval_unconstitutional", ] }, - "sources_to_database": { - "calendar_elected": "postgres_db", - }, "sources_to_platform_instance": { "calendar_elected": { + "database": "postgres_db", "env": "DEV", } }, @@ -271,12 +270,11 @@ def test_fivetran_with_snowflake_dest_and_null_connector_user(pytestconfig, tmp_ "interval_unconstitutional", ] }, - "sources_to_database": { - "calendar_elected": "postgres_db", - }, "sources_to_platform_instance": { "calendar_elected": { + "platform": "postgres", "env": "DEV", + "database": "postgres_db", } }, }, @@ -374,3 +372,34 @@ def test_rename_destination_config(): match="destination_config is deprecated, please use snowflake_destination_config instead.", ): FivetranSourceConfig.parse_obj(config_dict) + + +def test_compat_sources_to_database() -> None: + config_dict = { + # We just need a valid fivetran_log_config to test the compat transformation. + "fivetran_log_config": { + "destination_platform": "snowflake", + "snowflake_destination_config": { + "account_id": "testid", + "warehouse": "test_wh", + "username": "test", + "password": "test@123", + "database": "test_database", + "role": "testrole", + "log_schema": "test", + }, + }, + "sources_to_database": {"calendar_elected": "my_db", "connector_2": "my_db_2"}, + "sources_to_platform_instance": {"calendar_elected": {"env": "DEV"}}, + } + + with pytest.warns( + ConfigurationWarning, + match=r"sources_to_database.*deprecated", + ): + config = FivetranSourceConfig.parse_obj(config_dict) + + assert config.sources_to_platform_instance == { + "calendar_elected": PlatformDetail(env="DEV", database="my_db"), + "connector_2": PlatformDetail(database="my_db_2"), + } From 7c8dba4bd9e6e2a3dbd71d940c97afb1b58aeaa9 Mon Sep 17 00:00:00 2001 From: haeniya Date: Thu, 24 Oct 2024 20:56:00 +0200 Subject: [PATCH 12/26] feat(ingest/tableau): support ingestion of access roles (#11157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yanik Häni Co-authored-by: Harshal Sheth --- .../types/container/ContainerType.java | 3 +- .../container/mappers/ContainerMapper.java | 7 + .../graphql/types/dataset/DatasetType.java | 2 +- .../types/dataset/mappers/DatasetMapper.java | 2 +- .../src/main/resources/entity.graphql | 5 + .../app/entity/container/ContainerEntity.tsx | 23 +- .../AccessManagement/AccessManagement.tsx | 11 +- .../src/graphql/container.graphql | 7 + .../java/com/linkedin/metadata/Constants.java | 2 +- .../ingestion/source/tableau/tableau.py | 90 +- ...leau_permission_ingestion_mces_golden.json | 44237 ++++++++++++++++ .../tableau/test_tableau_ingest.py | 74 +- .../src/main/resources/entity-registry.yml | 1 + 13 files changed, 44451 insertions(+), 13 deletions(-) create mode 100644 metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java index db44a5be8bdd3..37b021fcb1091 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java @@ -52,7 +52,8 @@ public class ContainerType Constants.DEPRECATION_ASPECT_NAME, Constants.DATA_PRODUCTS_ASPECT_NAME, Constants.STRUCTURED_PROPERTIES_ASPECT_NAME, - Constants.FORMS_ASPECT_NAME); + Constants.FORMS_ASPECT_NAME, + Constants.ACCESS_ASPECT_NAME); private static final Set FACET_FIELDS = ImmutableSet.of("origin", "platform"); private static final String ENTITY_NAME = "container"; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java index 2c0dc142bee3d..02357b3ddc349 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java @@ -2,6 +2,7 @@ import static com.linkedin.metadata.Constants.*; +import com.linkedin.common.Access; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.Deprecation; import com.linkedin.common.Forms; @@ -30,6 +31,7 @@ import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper; import com.linkedin.datahub.graphql.types.form.FormsMapper; import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; +import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper; import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; import com.linkedin.domain.Domains; @@ -105,6 +107,11 @@ public static Container map( context, new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn)); } + final EnvelopedAspect accessAspect = aspects.get(ACCESS_ASPECT_NAME); + if (accessAspect != null) { + result.setAccess(AccessMapper.map(new Access(accessAspect.getValue().data()), entityUrn)); + } + final EnvelopedAspect envelopedInstitutionalMemory = aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME); if (envelopedInstitutionalMemory != null) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 65b5d39e31569..6a3f9cb9b21f3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -86,7 +86,7 @@ public class DatasetType EMBED_ASPECT_NAME, DATA_PRODUCTS_ASPECT_NAME, BROWSE_PATHS_V2_ASPECT_NAME, - ACCESS_DATASET_ASPECT_NAME, + ACCESS_ASPECT_NAME, STRUCTURED_PROPERTIES_ASPECT_NAME, FORMS_ASPECT_NAME, SUB_TYPES_ASPECT_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index a7b5f6de0c183..0869463ba73ac 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -166,7 +166,7 @@ public Dataset apply( (dataset, dataMap) -> dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(dataMap)))); mappingHelper.mapToResult( - ACCESS_DATASET_ASPECT_NAME, + ACCESS_ASPECT_NAME, ((dataset, dataMap) -> dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn)))); mappingHelper.mapToResult( diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 16ef59114f86c..b3728aa676490 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -2837,6 +2837,11 @@ type Container implements Entity { """ exists: Boolean + """ + The Roles and the properties to access the container + """ + access: Access + """ Experimental API. For fetching extra entities that do not have custom UI code yet diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx index 9cd32cf33a013..89f9122c6287f 100644 --- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx +++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx @@ -8,7 +8,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab' import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection'; import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { useGetContainerQuery } from '../../../graphql/container.generated'; +import { useGetContainerQuery, GetContainerQuery } from '../../../graphql/container.generated'; import { ContainerEntitiesTab } from './ContainerEntitiesTab'; import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; @@ -17,6 +17,8 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; import { getDataProduct } from '../shared/utils'; import EmbeddedProfile from '../shared/embed/EmbeddedProfile'; +import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement'; +import { useAppConfig } from '../../useAppConfig'; /** * Definition of the DataHub Container entity. @@ -65,6 +67,8 @@ export class ContainerEntity implements Entity { useEntityQuery = useGetContainerQuery; + appconfig = useAppConfig; + renderProfile = (urn: string) => ( { name: 'Properties', component: PropertiesTab, }, + { + name: 'Access Management', + component: AccessManagement, + display: { + visible: (_, container: GetContainerQuery) => { + return ( + this.appconfig().config.featureFlags.showAccessManagement && + !!container?.container?.access + ); + }, + enabled: (_, container: GetContainerQuery) => { + const accessAspect = container?.container?.access; + const rolesList = accessAspect?.roles; + return !!accessAspect && !!rolesList && rolesList.length > 0; + }, + }, + }, ]} sidebarSections={this.getSidebarSections()} /> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx index 5de532d17a676..f5789447d953d 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx @@ -3,8 +3,8 @@ import styled from 'styled-components'; import { Button, Table } from 'antd'; import { SpinProps } from 'antd/es/spin'; import { LoadingOutlined } from '@ant-design/icons'; -import { useBaseEntity } from '../../../EntityContext'; -import { GetDatasetQuery, useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated'; +import { useEntityData } from '../../../EntityContext'; +import { useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated'; import { handleAccessRoles } from './utils'; import AccessManagerDescription from './AccessManagerDescription'; @@ -60,11 +60,12 @@ const AccessButton = styled(Button)` `; export default function AccessManagement() { - const baseEntity = useBaseEntity(); + const { entityData } = useEntityData(); + const entityUrn = (entityData as any)?.urn; const { data: externalRoles, loading: isLoading } = useGetExternalRolesQuery({ - variables: { urn: baseEntity?.dataset?.urn as string }, - skip: !baseEntity?.dataset?.urn, + variables: { urn: entityUrn as string }, + skip: !entityUrn, }); const columns = [ diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql index 4b3ecfe8aaaff..88ed7582e62c5 100644 --- a/datahub-web-react/src/graphql/container.graphql +++ b/datahub-web-react/src/graphql/container.graphql @@ -59,6 +59,13 @@ query getContainer($urn: String!) { status { removed } + access { + roles { + role { + urn + } + } + } autoRenderAspects: aspects(input: { autoRenderOnly: true }) { ...autoRenderAspectFields } diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index c280324e10f47..295e48ecb748d 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -277,7 +277,7 @@ public class Constants { // ExternalRoleMetadata public static final String ROLE_ENTITY_NAME = "role"; - public static final String ACCESS_DATASET_ASPECT_NAME = "access"; + public static final String ACCESS_ASPECT_NAME = "access"; public static final String ROLE_KEY = "roleKey"; public static final String ROLE_PROPERTIES_ASPECT_NAME = "roleProperties"; public static final String ROLE_ACTORS_ASPECT_NAME = "actors"; diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 2c17a5a322f05..c1d899f11f2e1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -1,3 +1,4 @@ +import json import logging import re import time @@ -26,6 +27,8 @@ from pydantic.fields import Field from requests.adapters import HTTPAdapter from tableauserverclient import ( + GroupItem, + PermissionsRule, PersonalAccessTokenAuth, Server, ServerResponseError, @@ -216,6 +219,11 @@ class TableauConnectionConfig(ConfigModel): description="Whether to verify SSL certificates. If using self-signed certificates, set to false or provide the path to the .pem certificate bundle.", ) + session_trust_env: bool = Field( + False, + description="Configures the trust_env property in the requests session. If set to false (default value) it will bypass proxy settings. See https://requests.readthedocs.io/en/latest/api/#requests.Session.trust_env for more information.", + ) + extract_column_level_lineage: bool = Field( True, description="When enabled, extracts column-level lineage from Tableau Datasources", @@ -265,8 +273,7 @@ def make_tableau_client(self, site: str) -> Server: }, ) - # From https://stackoverflow.com/a/50159273/5004662. - server._session.trust_env = False + server._session.trust_env = self.session_trust_env # Setup request retries. adapter = HTTPAdapter( @@ -298,6 +305,23 @@ def make_tableau_client(self, site: str) -> Server: ) from e +class PermissionIngestionConfig(ConfigModel): + enable_workbooks: bool = Field( + default=True, + description="Whether or not to enable group permission ingestion for workbooks. " + "Default: True", + ) + + group_name_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Filter for Tableau group names when ingesting group permissions. " + "For example, you could filter for groups that include the term 'Consumer' in their name by adding '^.*Consumer$' to the allow list." + "By default, all groups will be ingested. " + "You can both allow and deny groups based on their name using their name, or a Regex pattern. " + "Deny patterns always take precedence over allow patterns. ", + ) + + class TableauConfig( DatasetLineageProviderConfigBase, StatefulIngestionConfigBase, @@ -459,6 +483,11 @@ class TableauConfig( description="When enabled, sites are added as containers and therefore visible in the folder structure within Datahub.", ) + permission_ingestion: Optional[PermissionIngestionConfig] = Field( + default=None, + description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.", + ) + # pre = True because we want to take some decision before pydantic initialize the configuration to default values @root_validator(pre=True) def projects_backward_compatibility(cls, values: Dict) -> Dict: @@ -732,6 +761,8 @@ def __init__( self.workbook_project_map: Dict[str, str] = {} self.datasource_project_map: Dict[str, str] = {} + self.group_map: Dict[str, GroupItem] = {} + # This map keeps track of the database server connection hostnames. self.database_server_hostname_map: Dict[str, str] = {} # This list keeps track of sheets in workbooks so that we retrieve those @@ -2803,6 +2834,18 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni f"Could not load project hierarchy for workbook {workbook_name}({workbook_id}). Please check permissions." ) + custom_props = None + if ( + self.config.permission_ingestion + and self.config.permission_ingestion.enable_workbooks + ): + logger.debug(f"Ingest access roles of workbook-id='{workbook.get(c.LUID)}'") + workbook_instance = self.server.workbooks.get_by_id(workbook.get(c.LUID)) + self.server.workbooks.populate_permissions(workbook_instance) + custom_props = self._create_workbook_properties( + workbook_instance.permissions + ) + yield from gen_containers( container_key=workbook_container_key, name=workbook.get(c.NAME) or "", @@ -2811,6 +2854,7 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni sub_types=[BIContainerSubTypes.TABLEAU_WORKBOOK], owner_urn=owner_urn, external_url=workbook_external_url, + extra_properties=custom_props, tags=tags, ) @@ -3168,11 +3212,53 @@ def emit_site_container(self): sub_types=[c.SITE], ) + def _fetch_groups(self): + for group in TSC.Pager(self.server.groups): + self.group_map[group.id] = group + + def _get_allowed_capabilities(self, capabilities: Dict[str, str]) -> List[str]: + if not self.config.permission_ingestion: + return [] + + allowed_capabilities = [ + key for key, value in capabilities.items() if value == "Allow" + ] + return allowed_capabilities + + def _create_workbook_properties( + self, permissions: List[PermissionsRule] + ) -> Optional[Dict[str, str]]: + if not self.config.permission_ingestion: + return None + + groups = [] + for rule in permissions: + if rule.grantee.tag_name == "group": + group = self.group_map.get(rule.grantee.id) + if not group or not group.name: + logger.debug(f"Group {rule.grantee.id} not found in group map.") + continue + if not self.config.permission_ingestion.group_name_pattern.allowed( + group.name + ): + logger.info( + f"Skip permission '{group.name}' as it's excluded in group_name_pattern." + ) + continue + + capabilities = self._get_allowed_capabilities(rule.capabilities) + groups.append({"group": group.name, "capabilities": capabilities}) + + return {"permissions": json.dumps(groups)} if len(groups) > 0 else None + def ingest_tableau_site(self): # Initialise the dictionary to later look-up for chart and dashboard stat if self.config.extract_usage_stats: self._populate_usage_stat_registry() + if self.config.permission_ingestion: + self._fetch_groups() + # Populate the map of database names and database hostnames to be used later to map # databases to platform instances. if self.config.database_hostname_to_platform_instance_map: diff --git a/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json new file mode 100644 index 0000000000000..cfc66af6f4569 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json @@ -0,0 +1,44237 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9", + "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea", + "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf", + "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c", + "permissions": "[{\"group\": \"AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer\", \"capabilities\": [\"Read\", \"ViewComments\"]}]" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems", + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_group_ingest" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 3798df07000c8..edfc41616e44b 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -8,14 +8,16 @@ import pytest from freezegun import freeze_time from requests.adapters import ConnectionError -from tableauserverclient import Server +from tableauserverclient import PermissionsRule, Server from tableauserverclient.models import ( DatasourceItem, + GroupItem, ProjectItem, SiteItem, ViewItem, WorkbookItem, ) +from tableauserverclient.models.reference_item import ResourceReference from datahub.configuration.source_common import DEFAULT_ENV from datahub.emitter.mce_builder import make_schema_field_urn @@ -132,6 +134,43 @@ def side_effect_project_data(*arg, **kwargs): return [project1, project2, project3, project4], mock_pagination +def side_effect_group_data(*arg, **kwargs): + mock_pagination = mock.MagicMock() + mock_pagination.total_available = None + + group1: GroupItem = GroupItem( + name="AB_XY00-Tableau-Access_A_123_PROJECT_XY_Consumer" + ) + group1._id = "79d02655-88e5-45a6-9f9b-eeaf5fe54903-group1" + group2: GroupItem = GroupItem( + name="AB_XY00-Tableau-Access_A_123_PROJECT_XY_Analyst" + ) + group2._id = "79d02655-88e5-45a6-9f9b-eeaf5fe54903-group2" + + return [group1, group2], mock_pagination + + +def side_effect_workbook_permissions(*arg, **kwargs): + project_capabilities1 = {"Read": "Allow", "ViewComments": "Allow"} + reference: ResourceReference = ResourceReference( + id_="79d02655-88e5-45a6-9f9b-eeaf5fe54903-group1", tag_name="group" + ) + rule1 = PermissionsRule(grantee=reference, capabilities=project_capabilities1) + + project_capabilities2 = { + "Read": "Allow", + "ViewComments": "Allow", + "Delete": "Allow", + "Write": "Allow", + } + reference2: ResourceReference = ResourceReference( + id_="79d02655-88e5-45a6-9f9b-eeaf5fe54903-group2", tag_name="group" + ) + rule2 = PermissionsRule(grantee=reference2, capabilities=project_capabilities2) + + return [rule1, rule2] + + def side_effect_site_data(*arg, **kwargs): mock_pagination = mock.MagicMock() mock_pagination.total_available = None @@ -249,8 +288,10 @@ def mock_sdk_client( mock_client.views = mock.Mock() mock_client.projects = mock.Mock() mock_client.sites = mock.Mock() + mock_client.groups = mock.Mock() mock_client.projects.get.side_effect = side_effect_project_data + mock_client.groups.get.side_effect = side_effect_group_data mock_client.sites.get.side_effect = side_effect_site_data mock_client.sites.get_by_id.side_effect = side_effect_site_get_by_id @@ -260,6 +301,11 @@ def mock_sdk_client( mock_client.workbooks = mock.Mock() mock_client.workbooks.get.side_effect = side_effect_workbook_data + workbook_mock = mock.create_autospec(WorkbookItem, instance=True) + type(workbook_mock).permissions = mock.PropertyMock( + return_value=side_effect_workbook_permissions() + ) + mock_client.workbooks.get_by_id.return_value = workbook_mock mock_client.views.get.side_effect = side_effect_usage_stat mock_client.auth.sign_in.return_value = None @@ -1154,6 +1200,32 @@ def test_site_name_pattern(pytestconfig, tmp_path, mock_datahub_graph): ) +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_permission_ingestion_mces.json" + golden_file_name: str = "tableau_permission_ingestion_mces_golden.json" + + new_pipeline_config: Dict[Any, Any] = { + **config_source_default, + "permission_ingestion": { + "enable_workbooks": True, + "group_name_pattern": {"allow": ["^.*_Consumer$"]}, + }, + } + tableau_ingest_common( + pytestconfig, + tmp_path, + mock_data(), + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_pipeline_config, + pipeline_name="test_tableau_group_ingest", + ) + + @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_graph): diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index ec9c3fee1c404..d3c6c8170f932 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -248,6 +248,7 @@ entities: - structuredProperties - forms - testResults + - access - name: tag category: core keyAspect: tagKey From ffefb9d8fe76520994ac5a713972d2ddb83ec1c6 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Thu, 24 Oct 2024 15:40:56 -0500 Subject: [PATCH 13/26] fix(logging): minor modifications for logging (#11703) --- li-utils/src/main/java/com/linkedin/metadata/Constants.java | 2 +- .../metadata/service/UpdateGraphIndicesService.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 295e48ecb748d..b29741b732224 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -460,7 +460,7 @@ public class Constants { // Logging MDC public static final String MDC_ENTITY_URN = "entityUrn"; - public static final String MDC_ASPECT_NAME = ""; + public static final String MDC_ASPECT_NAME = "aspectName"; public static final String MDC_ENTITY_TYPE = "entityType"; public static final String MDC_CHANGE_TYPE = "changeType"; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java index 3829855caa08b..ef7f681a81539 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java @@ -388,24 +388,24 @@ private void updateGraphServiceDiff( // Remove any old edges that no longer exist first if (!subtractiveDifference.isEmpty()) { log.debug("Removing edges: {}", subtractiveDifference); + subtractiveDifference.forEach(graphService::removeEdge); MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_REMOVE_METRIC) .inc(subtractiveDifference.size()); - subtractiveDifference.forEach(graphService::removeEdge); } // Then add new edges if (!additiveDifference.isEmpty()) { log.debug("Adding edges: {}", additiveDifference); + additiveDifference.forEach(graphService::addEdge); MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_ADD_METRIC) .inc(additiveDifference.size()); - additiveDifference.forEach(graphService::addEdge); } // Then update existing edges if (!mergedEdges.isEmpty()) { log.debug("Updating edges: {}", mergedEdges); - MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_UPDATE_METRIC).inc(mergedEdges.size()); mergedEdges.forEach(graphService::upsertEdge); + MetricUtils.counter(this.getClass(), GRAPH_DIFF_MODE_UPDATE_METRIC).inc(mergedEdges.size()); } } From 87fa5b89e8e5bcca668f4c2a6675c791c146a95a Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:56:55 +0530 Subject: [PATCH 14/26] feat: multi-query lineage for temp upstreams (#11708) --- .../ingestion/source/redshift/lineage.py | 4 +- .../source/snowflake/snowflake_queries.py | 29 +++-- .../sql_parsing/sql_parsing_aggregator.py | 89 +++++++++----- .../bigquery_queries_mcps_golden.json | 10 +- .../test_multistep_temp_table.json | 16 +-- ..._overlapping_inserts_from_temp_tables.json | 32 ++--- ...lineage_via_temp_table_disordered_add.json | 8 +- .../aggregator_goldens/test_table_rename.json | 47 ++++++++ .../test_table_rename_with_temp.json | 109 ++++++++++++------ .../aggregator_goldens/test_table_swap.json | 47 ++++++++ .../test_table_swap_with_temp.json | 105 ++++++++++++++--- .../aggregator_goldens/test_temp_table.json | 32 ++--- .../unit/sql_parsing/test_sql_aggregator.py | 105 ++++++++++++++++- 13 files changed, 493 insertions(+), 140 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py index 192c97bbee348..4b3d238a13261 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/lineage.py @@ -894,7 +894,9 @@ def _process_table_renames( env=self.config.env, ) - table_renames[new_urn] = TableRename(prev_urn, new_urn, query_text) + table_renames[new_urn] = TableRename( + prev_urn, new_urn, query_text, timestamp=rename_row.start_time + ) # We want to generate lineage for the previous name too. all_tables[database][schema].add(prev_name) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index 33dc1388ff0e1..2d2bdc50467c6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -354,15 +354,23 @@ def _parse_audit_log_row( object_modified_by_ddl = res["object_modified_by_ddl"] if object_modified_by_ddl and not objects_modified: - ddl_entry: Optional[Union[TableRename, TableSwap]] = None + known_ddl_entry: Optional[Union[TableRename, TableSwap]] = None with self.structured_reporter.report_exc( "Error fetching ddl lineage from Snowflake" ): - ddl_entry = self.parse_ddl_query( - res["query_text"], object_modified_by_ddl + known_ddl_entry = self.parse_ddl_query( + res["query_text"], + res["session_id"], + res["query_start_time"], + object_modified_by_ddl, ) - return ddl_entry - + if known_ddl_entry: + return known_ddl_entry + elif direct_objects_accessed: + # Unknown ddl relevant for usage. We want to continue execution here + pass + else: + return None upstreams = [] column_usage = {} @@ -459,8 +467,13 @@ def _parse_audit_log_row( return entry def parse_ddl_query( - self, query: str, object_modified_by_ddl: dict + self, + query: str, + session_id: str, + timestamp: datetime, + object_modified_by_ddl: dict, ) -> Optional[Union[TableRename, TableSwap]]: + timestamp = timestamp.astimezone(timezone.utc) if object_modified_by_ddl[ "operationType" ] == "ALTER" and object_modified_by_ddl["properties"].get("swapTargetName"): @@ -476,7 +489,7 @@ def parse_ddl_query( ) ) - return TableSwap(urn1, urn2, query) + return TableSwap(urn1, urn2, query, session_id, timestamp) elif object_modified_by_ddl[ "operationType" ] == "RENAME_TABLE" and object_modified_by_ddl["properties"].get("objectName"): @@ -492,7 +505,7 @@ def parse_ddl_query( ) ) - return TableRename(original_un, new_urn, query) + return TableRename(original_un, new_urn, query, session_id, timestamp) else: self.report.num_ddl_queries_dropped += 1 return None diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 9aab33bf09f05..255d4c6e6bb74 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -180,6 +180,7 @@ class TableRename: new_urn: UrnStr query: Optional[str] = None session_id: str = _MISSING_SESSION_ID + timestamp: Optional[datetime] = None @dataclasses.dataclass @@ -188,6 +189,7 @@ class TableSwap: urn2: UrnStr query: Optional[str] = None session_id: str = _MISSING_SESSION_ID + timestamp: Optional[datetime] = None def id(self) -> str: # TableSwap(A,B) is same as TableSwap(B,A) @@ -444,8 +446,7 @@ def __init__( # Map of session ID -> {temp table name -> query id} # Needs to use the query_map to find the info about the query. - # This assumes that a temp table is created at most once per session. - self._temp_lineage_map = FileBackedDict[Dict[UrnStr, QueryId]]( + self._temp_lineage_map = FileBackedDict[Dict[UrnStr, OrderedSet[QueryId]]]( shared_connection=self._shared_connection, tablename="temp_lineage_map" ) self._exit_stack.push(self._temp_lineage_map) @@ -903,12 +904,10 @@ def add_preparsed_query( parsed.query_type.is_create() and parsed.query_type_props.get("temporary") ) + or self.is_temp_table(out_table) or ( - self.is_temp_table(out_table) - or ( - require_out_table_schema - and not self._schema_resolver.has_urn(out_table) - ) + require_out_table_schema + and not self._schema_resolver.has_urn(out_table) ) ): # Infer the schema of the output table and track it for later. @@ -916,9 +915,9 @@ def add_preparsed_query( self._inferred_temp_schemas[query_fingerprint] = parsed.inferred_schema # Also track the lineage for the temp table, for merging purposes later. - self._temp_lineage_map.for_mutation(parsed.session_id, {})[ - out_table - ] = query_fingerprint + self._temp_lineage_map.for_mutation(parsed.session_id, {}).setdefault( + out_table, OrderedSet() + ).add(query_fingerprint) # Also update schema resolver for missing session id if parsed.session_id == _MISSING_SESSION_ID and parsed.inferred_schema: @@ -962,6 +961,8 @@ def add_table_rename( downstream_urn=table_rename.new_urn, upstream_urn=table_rename.original_urn, ), + session_id=table_rename.session_id, + timestamp=table_rename.timestamp, ) ) @@ -996,6 +997,11 @@ def add_table_swap(self, table_swap: TableSwap) -> None: f"\nalter table {table1} swap with {table2}", upstreams=[table_swap.urn1], downstream=table_swap.urn2, + column_lineage=self._generate_identity_column_lineage( + upstream_urn=table_swap.urn1, downstream_urn=table_swap.urn2 + ), + session_id=table_swap.session_id, + timestamp=table_swap.timestamp, ) ) @@ -1007,6 +1013,11 @@ def add_table_swap(self, table_swap: TableSwap) -> None: f"alter table {table2} swap with {table1}", upstreams=[table_swap.urn2], downstream=table_swap.urn1, + column_lineage=self._generate_identity_column_lineage( + upstream_urn=table_swap.urn2, downstream_urn=table_swap.urn1 + ), + session_id=table_swap.session_id, + timestamp=table_swap.timestamp, ) ) @@ -1018,10 +1029,13 @@ def _make_schema_resolver_for_session( schema_resolver = self._missing_session_schema_resolver elif session_id in self._temp_lineage_map: temp_table_schemas: Dict[str, Optional[List[models.SchemaFieldClass]]] = {} - for temp_table_urn, query_id in self._temp_lineage_map[session_id].items(): - temp_table_schemas[temp_table_urn] = self._inferred_temp_schemas.get( - query_id - ) + for temp_table_urn, query_ids in self._temp_lineage_map[session_id].items(): + for query_id in query_ids: + temp_table_schemas[ + temp_table_urn + ] = self._inferred_temp_schemas.get(query_id) + if temp_table_schemas: + break if temp_table_schemas: schema_resolver = self._schema_resolver.with_temp_tables( @@ -1515,6 +1529,13 @@ class QueryLineageInfo: column_lineage: List[ColumnLineageInfo] confidence_score: float + def _merge_lineage_from(self, other_query: "QueryLineageInfo") -> None: + self.upstreams += other_query.upstreams + self.column_lineage += other_query.column_lineage + self.confidence_score = min( + self.confidence_score, other_query.confidence_score + ) + def _recurse_into_query( query: QueryMetadata, recursion_path: List[QueryId] ) -> QueryLineageInfo: @@ -1531,15 +1552,24 @@ def _recurse_into_query( # Find all the temp tables that this query depends on. temp_upstream_queries: Dict[UrnStr, QueryLineageInfo] = {} for upstream in query.upstreams: - upstream_query_id = self._temp_lineage_map.get(session_id, {}).get( + upstream_query_ids = self._temp_lineage_map.get(session_id, {}).get( upstream ) - if upstream_query_id: - upstream_query = self._query_map.get(upstream_query_id) - if upstream_query: - temp_upstream_queries[upstream] = _recurse_into_query( - upstream_query, recursion_path - ) + if upstream_query_ids: + for upstream_query_id in upstream_query_ids: + upstream_query = self._query_map.get(upstream_query_id) + if upstream_query: + temp_query_lineage_info = _recurse_into_query( + upstream_query, recursion_path + ) + if upstream in temp_upstream_queries: + temp_upstream_queries[upstream]._merge_lineage_from( + temp_query_lineage_info + ) + else: + temp_upstream_queries[ + upstream + ] = temp_query_lineage_info # Compute merged upstreams. new_upstreams = OrderedSet[UrnStr]() @@ -1605,7 +1635,17 @@ def _recurse_into_query( # - Update the lineage info # - Update the query text to combine the queries - composite_query_id = self._composite_query_id(composed_of_queries) + ordered_queries = [ + self._query_map[query_id] for query_id in reversed(composed_of_queries) + ] + if all(q.latest_timestamp for q in ordered_queries): + ordered_queries = sorted( + ordered_queries, + key=lambda query: make_ts_millis(query.latest_timestamp) or 0, + ) + composite_query_id = self._composite_query_id( + [q.query_id for q in ordered_queries] + ) composed_of_queries_truncated: LossyList[str] = LossyList() for query_id in composed_of_queries: composed_of_queries_truncated.append(query_id) @@ -1614,10 +1654,7 @@ def _recurse_into_query( ] = composed_of_queries_truncated merged_query_text = ";\n\n".join( - [ - self._query_map[query_id].formatted_query_string - for query_id in reversed(composed_of_queries) - ] + [q.formatted_query_string for q in ordered_queries] ) resolved_query = dataclasses.replace( diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json index 1b79e8464c05f..9a4881f016960 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json +++ b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_queries_mcps_golden.json @@ -193,7 +193,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:bigquery,gcp-staging.smoke_test_db.base_table,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b" + "query": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d" } ] } @@ -206,7 +206,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b", + "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -425,7 +425,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b", + "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -508,7 +508,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b", + "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -7890,7 +7890,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_0db44e02f671b69df68565346e9d2b68c7166fccf75bd494f34560bfa16c381b", + "entityUrn": "urn:li:query:composite_29c38b444a8740d9cc549168e2e0e3657fc00430520f615119bfc3e9fb94112d", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json index a9b5a3a7cbefa..83d81037a2136 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json @@ -18,7 +18,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream1,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" }, { "auditStamp": { @@ -31,7 +31,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.upstream2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" } ], "fineGrainedLineages": [ @@ -45,7 +45,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),a)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" }, { "upstreamType": "FIELD_SET", @@ -57,7 +57,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),b)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" }, { "upstreamType": "FIELD_SET", @@ -69,7 +69,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.prod_foo,PROD),c)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "query": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" } ] } @@ -77,7 +77,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45", + "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -100,7 +100,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45", + "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -130,7 +130,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45", + "entityUrn": "urn:li:query:composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json index 48aecb9015180..ed8009477bf4d 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json @@ -18,7 +18,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.customer,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "auditStamp": { @@ -31,7 +31,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.online_returns,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "auditStamp": { @@ -44,7 +44,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.online_survey,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "auditStamp": { @@ -57,7 +57,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.in_person_returns,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75" + "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45" } ], "fineGrainedLineages": [ @@ -71,7 +71,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75" + "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45" }, { "upstreamType": "FIELD_SET", @@ -83,7 +83,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_id)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "upstreamType": "FIELD_SET", @@ -95,7 +95,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),customer_email)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "upstreamType": "FIELD_SET", @@ -107,7 +107,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75" + "query": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45" }, { "upstreamType": "FIELD_SET", @@ -119,7 +119,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_date)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" }, { "upstreamType": "FIELD_SET", @@ -131,7 +131,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.all_returns,PROD),return_reason)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17" + "query": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8" } ] } @@ -139,7 +139,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17", + "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -162,7 +162,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17", + "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -198,7 +198,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17", + "entityUrn": "urn:li:query:composite_adc1c41c0ad37c643776d9d93d524e6c435a7e70633da1ce7e3222dda4bb9fb8", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -209,7 +209,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75", + "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -232,7 +232,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75", + "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -262,7 +262,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75", + "entityUrn": "urn:li:query:composite_6b5a11e96e3d2b742e4e4ec3310bb538d0f5c0c6496b84e4bfe0e8014d5f5b45", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json index 51a732e7a2494..915718525dbc7 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_lineage_via_temp_table_disordered_add.json @@ -18,7 +18,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f" + "query": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec" } ] } @@ -26,7 +26,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f", + "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -49,7 +49,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f", + "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -67,7 +67,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_39f4adf89c8ad4d6d307b628c82d8260e1c5cd7eb6fb3a8cbb437421f970c16f", + "entityUrn": "urn:li:query:composite_cb80bd1f17cc5d3d956e03a82d4266ec0dc698ca4333547281fb93bf67d832ec", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json index 19d19d1f56ae9..f6d781b356ee9 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json @@ -135,6 +135,44 @@ "type": "TRANSFORMED", "query": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_staging,PROD),c)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:a30d42497a737321ece461fa17344c3ba3588fdee736016acb59a00cec955a0c" + } ] } } @@ -175,6 +213,15 @@ }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json index 4e2eba778ab9a..abae5da02135d 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename_with_temp.json @@ -133,7 +133,33 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e" + "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + ], + "confidenceScore": 0.2, + "query": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332" } ] } @@ -141,7 +167,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", + "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -162,6 +188,44 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD),c)" + } + ] + } + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_2efc2a13ee673ccf7b195f8f2c0e4ba0570194d8200c3c20b1eb7e8ca4fb4332", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD)", @@ -181,7 +245,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703" } ], "fineGrainedLineages": [ @@ -195,7 +259,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),a)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703" }, { "upstreamType": "FIELD_SET", @@ -207,7 +271,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_downstream,PROD),b)" ], "confidenceScore": 0.2, - "query": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9" + "query": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703" } ] } @@ -215,36 +279,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_dep,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)" - } - ] - } - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:composite_4b8845f9fa02123e7501a9baf421175923514b4a4a85158b2df36f930bcdc10e", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:redshift" - } - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -267,7 +302,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -291,7 +326,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_ff126f9f14af4f1871d685a332dc4d71a507a6ca8b7a13e38e46cb58e0b1ecb9", + "entityUrn": "urn:li:query:composite_6c7b9cb477493c36e47135e9a699745ae77ef64de3a811aefb7a44c8c04de703", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json index 382bc8ee0281d..1992bced039be 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap.json @@ -135,6 +135,44 @@ "type": "TRANSFORMED", "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),b)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_swap,PROD),c)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:3865108263e5f0670e6506f5747392f8315a72039cbfde1c4be4dd9a71bdd500" + } ] } } @@ -252,6 +290,15 @@ }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)" } ] } diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json index 73084f49a8928..ba83917ca5c1a 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_swap_with_temp.json @@ -128,12 +128,40 @@ "actor": "urn:li:corpuser:_ingestion" }, "created": { - "time": 0, + "time": 40000, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce" + }, + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 40000, "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970" + "query": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce" } ] } @@ -141,22 +169,22 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "entityUrn": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "CREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info", + "value": "CREATE TABLE person_info_swap CLONE person_info;\n\nCREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nALTER TABLE dev.public.person_info_swap SWAP WITH dev.public.person_info", "language": "SQL" }, "source": "SYSTEM", "created": { - "time": 0, + "time": 40000, "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1707182625000, + "time": 40000, "actor": "urn:li:corpuser:_ingestion" } } @@ -164,17 +192,26 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "entityUrn": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),b)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),c)" } ] } @@ -182,7 +219,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_333307d51724d89c6c2761f83c9e54006c78565154857272f02c7c0849920970", + "entityUrn": "urn:li:query:composite_5f9c1232994672c5fb7621f8384f6600b6d4ed5acfccc4eb396fb446b3fb1bce", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -205,12 +242,40 @@ "actor": "urn:li:corpuser:_ingestion" }, "created": { - "time": 0, + "time": 50000, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80" + }, + { + "auditStamp": { + "time": 1707182625000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 50000, "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d" + "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD),a)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD),a)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD),a)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80" } ] } @@ -218,22 +283,22 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "CREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nCREATE TABLE person_info_backup AS\nSELECT\n *\nFROM person_info_swap", + "value": "CREATE TABLE person_info_swap CLONE person_info;\n\nCREATE TABLE person_info_incremental AS\nSELECT\n *\nFROM person_info_dep;\n\nINSERT INTO person_info_swap\nSELECT\n *\nFROM person_info_incremental;\n\nCREATE TABLE person_info_backup AS\nSELECT\n *\nFROM person_info_swap", "language": "SQL" }, "source": "SYSTEM", "created": { - "time": 0, + "time": 50000, "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1707182625000, + "time": 50000, "actor": "urn:li:corpuser:_ingestion" } } @@ -241,17 +306,23 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info,PROD)" + }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_dep,PROD)" }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,dev.public.person_info_backup,PROD),a)" } ] } @@ -259,7 +330,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_d5d07752cbe5b894085996dc22582334b88b5726c049a5a533c02c49a427766d", + "entityUrn": "urn:li:query:composite_49daa72ac1d22734879a6bed1224daa7f8c1293750d6d7b8a24a0aa0e9f74d80", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json index 743e2738fc70c..b348785d06431 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json @@ -133,7 +133,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.bar,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec" + "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817" } ], "fineGrainedLineages": [ @@ -147,7 +147,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),a)" ], "confidenceScore": 0.35, - "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec" + "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817" }, { "upstreamType": "FIELD_SET", @@ -160,7 +160,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD),c)" ], "confidenceScore": 0.35, - "query": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec" + "query": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817" } ] } @@ -168,7 +168,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec", + "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { @@ -191,7 +191,7 @@ }, { "entityType": "query", - "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec", + "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -213,6 +213,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_02ccf36422adc3d46b0707071c53ccc867a16444b62ff128f890266ff5399817", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session3,PROD)", @@ -238,17 +249,6 @@ } } }, -{ - "entityType": "query", - "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:redshift" - } - } -}, { "entityType": "query", "entityUrn": "urn:li:query:3e85e6f353c7fa33d6514cb090482852064d23df6491c9a8ae28be0d990a3c71", diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py index 45bbc19a2124f..eb64efd6693de 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py @@ -224,12 +224,11 @@ def test_multistep_temp_table(pytestconfig: pytest.Config) -> None: assert ( len( report.queries_with_temp_upstreams[ - "composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45" + "composite_48c238412066895ccad5d27f9425ce969b2c0633203627eb476d0c9e5357825a" ] ) == 4 ) - mce_helpers.check_goldens_stream( pytestconfig, outputs=mcps, @@ -523,6 +522,11 @@ def test_table_rename(pytestconfig: pytest.Config) -> None: generate_operations=False, ) + aggregator._schema_resolver.add_raw_schema_info( + DatasetUrn("redshift", "dev.public.foo").urn(), + {"a": "int", "b": "int", "c": "int"}, + ) + # Register that foo_staging is renamed to foo. aggregator.add_table_rename( TableRename( @@ -577,6 +581,11 @@ def test_table_rename_with_temp(pytestconfig: pytest.Config) -> None: is_temp_table=lambda x: "staging" in x.lower(), ) + aggregator._schema_resolver.add_raw_schema_info( + DatasetUrn("redshift", "dev.public.foo").urn(), + {"a": "int", "b": "int", "c": "int"}, + ) + # Register that foo_staging is renamed to foo. aggregator.add_table_rename( TableRename( @@ -631,6 +640,11 @@ def test_table_swap(pytestconfig: pytest.Config) -> None: generate_operations=False, ) + aggregator._schema_resolver.add_raw_schema_info( + DatasetUrn("snowflake", "dev.public.person_info").urn(), + {"a": "int", "b": "int", "c": "int"}, + ) + # Add an unrelated query. aggregator.add_observed_query( ObservedQuery( @@ -714,6 +728,11 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: is_temp_table=lambda x: "swap" in x.lower() or "incremental" in x.lower(), ) + aggregator._schema_resolver.add_raw_schema_info( + DatasetUrn("snowflake", "dev.public.person_info").urn(), + {"a": "int", "b": "int", "c": "int"}, + ) + # Add an unrelated query. aggregator.add_observed_query( ObservedQuery( @@ -730,6 +749,26 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: query_text="CREATE TABLE person_info_swap CLONE person_info;", upstreams=[DatasetUrn("snowflake", "dev.public.person_info").urn()], downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + session_id="xxx", + timestamp=_ts(10), + column_lineage=[ + ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_swap" + ).urn(), + column="a", + ), + upstreams=[ + ColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info" + ).urn(), + column="a", + ) + ], + ) + ], ) ) @@ -744,6 +783,26 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: downstream=DatasetUrn( "snowflake", "dev.public.person_info_incremental" ).urn(), + session_id="xxx", + timestamp=_ts(20), + column_lineage=[ + ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_incremental" + ).urn(), + column="a", + ), + upstreams=[ + ColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_dep" + ).urn(), + column="a", + ) + ], + ) + ], ) ) @@ -756,6 +815,26 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: DatasetUrn("snowflake", "dev.public.person_info_incremental").urn(), ], downstream=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + session_id="xxx", + timestamp=_ts(30), + column_lineage=[ + ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_swap" + ).urn(), + column="a", + ), + upstreams=[ + ColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_incremental" + ).urn(), + column="a", + ) + ], + ) + ], ) ) @@ -763,6 +842,8 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: TableSwap( urn1=DatasetUrn("snowflake", "dev.public.person_info").urn(), urn2=DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), + session_id="xxx", + timestamp=_ts(40), ) ) @@ -775,6 +856,26 @@ def test_table_swap_with_temp(pytestconfig: pytest.Config) -> None: DatasetUrn("snowflake", "dev.public.person_info_swap").urn(), ], downstream=DatasetUrn("snowflake", "dev.public.person_info_backup").urn(), + session_id="xxx", + timestamp=_ts(50), + column_lineage=[ + ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_backup" + ).urn(), + column="a", + ), + upstreams=[ + ColumnRef( + table=DatasetUrn( + "snowflake", "dev.public.person_info_swap" + ).urn(), + column="a", + ) + ], + ) + ], ) ) From 84d86ca23a081d08a2a5b83fda0ea8c1251512ca Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Fri, 25 Oct 2024 16:53:57 -0700 Subject: [PATCH 15/26] docs(managed/v0.3.6): Add additional release notes for 0.3.6.8 (#11715) --- docs/managed-datahub/release-notes/v_0_3_6.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/managed-datahub/release-notes/v_0_3_6.md b/docs/managed-datahub/release-notes/v_0_3_6.md index cf1c9ff4e8c6a..e513a46ace4c5 100644 --- a/docs/managed-datahub/release-notes/v_0_3_6.md +++ b/docs/managed-datahub/release-notes/v_0_3_6.md @@ -48,6 +48,15 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - Impact analysis and data product assets pages properly show results based on the currently selected view - The New Users section in the Analytics page now tracks actual new users instead of last login - Data Contracts builder now shows custom assertions as expected + - "View in Platform" buttons link location fixed + - Compact search cards unnecessary title truncation fixed + - Domain contents summary updates after domain is changed, without refresh + - Incidents tab updates after an incident is created, updated, or resolved, without refresh + - On lineage visualization, fix lineage expansion through transformation nodes while hiding transformations + - On lineage visualization, fix positioning of query nodes on lineage expansion + - On lineage visualization, cycles in graphs should no longer result in disappearing edges + - Remote executor cleans up temporary files to avoid disk memory leak + - Fixed hover cards when using Compact Card View - Product changes - Show total (nested) asset count on home page Domain cards @@ -68,6 +77,11 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - Add "Last Updated (in Source)" filter to search - Add "View In \[Source]" button to lineage visualization sidebar - Send test notification button will always be visible in the subscriptions side bar if the user has set their memberID + - On sibling combined entity page, viewing different schema versions no longer supported + - On sibling combined entity page, change history drawer reflects changes to both siblings + - On sibling combined entity sidebar, lineage summary is hidden to avoid confusion + - For siblings combined entity page or search / hover card, multiple "View in Platform" icons should appear if applicable + - Search cards configured to better handle smaller widths - Ingestion changes From 6770ee85354ffd546168627181a9ec88a14bd6d9 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 25 Oct 2024 22:29:19 -0500 Subject: [PATCH 16/26] gradle(profiles): create compose base variable (#11716) --- docker/build.gradle | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/build.gradle b/docker/build.gradle index cdf2d1271d6e8..6d53d593186dd 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -8,6 +8,8 @@ import com.avast.gradle.dockercompose.tasks.ComposeDownForced apply from: "../gradle/versioning/versioning.gradle" ext { + compose_base = "profiles/docker-compose.yml" + backend_profile_modules = [ ':docker:elasticsearch-setup', ':docker:mysql-setup', @@ -23,7 +25,7 @@ ext { debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job', ':metadata-jobs:mae-consumer-job'] - compose_args = ['-f', 'profiles/docker-compose.yml'] + compose_args = ['-f', compose_base] debug_reloadable = [ 'datahub-gms-debug', 'system-update-debug', @@ -68,7 +70,7 @@ dockerCompose { environment.put 'DATAHUB_VERSION', "v${version}" environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally - useComposeFiles = ['profiles/docker-compose.yml'] + useComposeFiles = [compose_base] projectName = 'datahub' projectNamePrefix = '' buildBeforeUp = false @@ -86,7 +88,7 @@ dockerCompose { environment.put 'DATAHUB_VERSION', "v${version}" environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally - useComposeFiles = ['profiles/docker-compose.yml'] + useComposeFiles = [compose_base] projectName = 'datahub' projectNamePrefix = '' buildBeforeUp = false @@ -113,7 +115,7 @@ dockerCompose { // disabled for spark-lineage smoke-test environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env" - useComposeFiles = ['profiles/docker-compose.yml'] + useComposeFiles = [compose_base] projectName = 'datahub' projectNamePrefix = '' buildBeforeUp = false @@ -133,7 +135,7 @@ dockerCompose { } environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally - useComposeFiles = ['profiles/docker-compose.yml'] + useComposeFiles = [compose_base] projectName = 'datahub' projectNamePrefix = '' buildBeforeUp = false @@ -146,7 +148,7 @@ dockerCompose { isRequiredBy(tasks.named('quickstartStorage')) composeAdditionalArgs = ['--profile', 'quickstart-storage'] - useComposeFiles = ['profiles/docker-compose.yml'] + useComposeFiles = [compose_base] projectName = 'datahub' projectNamePrefix = '' buildBeforeUp = false From a6e1624f489b9e4886d87cbb36beaabb12fe8f56 Mon Sep 17 00:00:00 2001 From: Davi Arnaut Date: Sat, 26 Oct 2024 00:10:40 -0400 Subject: [PATCH 17/26] fix(log): reduce log volume for ingestion and consumers (#11714) --- .../linkedin/metadata/entity/EntityServiceImpl.java | 10 ++++++---- .../search/elasticsearch/ElasticSearchService.java | 2 +- .../com/linkedin/metadata/kafka/MCLKafkaListener.java | 2 +- .../metadata/resources/entity/AspectResource.java | 9 --------- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 9f608be4f3d18..9337ea3c2b6f7 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -164,6 +164,8 @@ public class EntityServiceImpl implements EntityService { private final Integer ebeanMaxTransactionRetry; private final boolean enableBrowseV2; + private static final long DB_TIMER_LOG_THRESHOLD_MS = 50; + @Getter private final Map, ThrottleEvent> throttleEvents = new ConcurrentHashMap<>(); @@ -997,10 +999,10 @@ private List ingestAspectsToLocalDB( if (txContext != null) { txContext.commitAndContinue(); } - long took = ingestToLocalDBTimer.stop(); - log.info( - "Ingestion of aspects batch to database took {} ms", - TimeUnit.NANOSECONDS.toMillis(took)); + long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop()); + if (took > DB_TIMER_LOG_THRESHOLD_MS) { + log.info("Ingestion of aspects batch to database took {} ms", took); + } // Retention optimization and tx if (retentionService != null) { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 6001e2f6e660f..261ec127d5497 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -112,7 +112,7 @@ public void appendRunId( @Nullable String runId) { final String docId = indexBuilders.getIndexConvention().getEntityDocumentId(urn); - log.info( + log.debug( "Appending run id for entity name: {}, doc id: {}, run id: {}", entityName, docId, runId); esWriteDAO.applyScriptUpdate( opContext, diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java index ff929603514c5..a2d59023ba5ce 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListener.java @@ -95,7 +95,7 @@ public void consume(final ConsumerRecord consumerRecord) // Here - plug in additional "custom processor hooks" for (MetadataChangeLogHook hook : this.hooks) { - log.info( + log.debug( "Invoking MCL hook {} for urn: {}", hook.getClass().getSimpleName(), event.getEntityUrn()); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 634e8d32cc722..63b607f8c9967 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -243,13 +243,6 @@ public Task ingestProposal( @ActionParam(PARAM_PROPOSAL) @Nonnull MetadataChangeProposal metadataChangeProposal, @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) throws URISyntaxException { - - String urn = metadataChangeProposal.getEntityUrn() != null ? metadataChangeProposal.getEntityUrn().toString() : - java.util.Optional.ofNullable(metadataChangeProposal.getEntityKeyAspect()).orElse(new GenericAspect()) - .getValue().asString(StandardCharsets.UTF_8); - String proposedValue = java.util.Optional.ofNullable(metadataChangeProposal.getAspect()).orElse(new GenericAspect()) - .getValue().asString(StandardCharsets.UTF_8); - final boolean asyncBool; if (UNSET.equals(async)) { asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); @@ -266,8 +259,6 @@ public Task ingestProposalBatch( @ActionParam(PARAM_PROPOSALS) @Nonnull MetadataChangeProposal[] metadataChangeProposals, @ActionParam(PARAM_ASYNC) @Optional(UNSET) String async) throws URISyntaxException { - log.info("INGEST PROPOSAL BATCH proposals: {}", Arrays.asList(metadataChangeProposals)); - final boolean asyncBool; if (UNSET.equals(async)) { asyncBool = Boolean.parseBoolean(System.getenv(ASYNC_INGEST_DEFAULT_NAME)); From abe88845976053a4f6356f08fc546efdc4c0bae7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:03:19 -0500 Subject: [PATCH 18/26] misc(gradle): project name parameter (#11717) --- docker/build.gradle | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/build.gradle b/docker/build.gradle index 6d53d593186dd..25e3dc12036ef 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -9,6 +9,7 @@ apply from: "../gradle/versioning/versioning.gradle" ext { compose_base = "profiles/docker-compose.yml" + project_name = "datahub" backend_profile_modules = [ ':docker:elasticsearch-setup', @@ -71,7 +72,7 @@ dockerCompose { environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = [compose_base] - projectName = 'datahub' + projectName = project_name projectNamePrefix = '' buildBeforeUp = false buildBeforePull = false @@ -89,7 +90,7 @@ dockerCompose { environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = [compose_base] - projectName = 'datahub' + projectName = project_name projectNamePrefix = '' buildBeforeUp = false buildBeforePull = false @@ -116,7 +117,7 @@ dockerCompose { environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env" useComposeFiles = [compose_base] - projectName = 'datahub' + projectName = project_name projectNamePrefix = '' buildBeforeUp = false buildBeforePull = false @@ -136,7 +137,7 @@ dockerCompose { environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally useComposeFiles = [compose_base] - projectName = 'datahub' + projectName = project_name projectNamePrefix = '' buildBeforeUp = false buildBeforePull = false @@ -149,7 +150,7 @@ dockerCompose { composeAdditionalArgs = ['--profile', 'quickstart-storage'] useComposeFiles = [compose_base] - projectName = 'datahub' + projectName = project_name projectNamePrefix = '' buildBeforeUp = false buildBeforePull = false From 0e4b3c15c9aba5b42cd97fa7ce72458fedd49cc7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:05:16 -0500 Subject: [PATCH 19/26] refactor(datahub-frontend): upgrade frontend pac4j (#11709) --- build.gradle | 30 ++- datahub-frontend/app/auth/AuthModule.java | 91 +++---- datahub-frontend/app/auth/AuthUtils.java | 1 + datahub-frontend/app/auth/CookieConfigs.java | 18 +- datahub-frontend/app/auth/JAASConfigs.java | 6 +- .../app/auth/NativeAuthenticationConfigs.java | 12 +- datahub-frontend/app/auth/sso/SsoConfigs.java | 53 ++--- datahub-frontend/app/auth/sso/SsoManager.java | 38 +-- .../app/auth/sso/SsoProvider.java | 9 +- .../sso/oidc/OidcAuthorizationGenerator.java | 21 +- .../app/auth/sso/oidc/OidcCallbackLogic.java | 224 ++++++++++++------ .../app/auth/sso/oidc/OidcConfigs.java | 13 +- .../app/auth/sso/oidc/OidcProvider.java | 71 +++--- .../sso/oidc/OidcResponseErrorHandler.java | 19 +- .../oidc/custom/CustomOidcAuthenticator.java | 94 +++++--- .../sso/oidc/custom/CustomOidcClient.java | 37 ++- .../CustomOidcRedirectionActionBuilder.java | 21 +- .../app/client/KafkaTrackingProducer.java | 27 ++- .../app/config/ConfigurationProvider.java | 3 - .../app/controllers/Application.java | 120 +++++----- .../controllers/AuthenticationController.java | 154 ++++++------ .../controllers/CentralLogoutController.java | 13 +- .../app/controllers/RedirectController.java | 5 +- .../controllers/SsoCallbackController.java | 88 +++---- .../app/controllers/TrackingController.java | 22 +- datahub-frontend/build.gradle | 6 + datahub-frontend/conf/application.conf | 1 + datahub-frontend/play.gradle | 40 ++-- .../test/app/ApplicationTest.java | 196 ++++++++++++--- .../test/oidc/OidcCallbackLogicTest.java | 91 +++---- .../test/security/OidcConfigurationTest.java | 19 +- 31 files changed, 943 insertions(+), 600 deletions(-) diff --git a/build.gradle b/build.gradle index 67968ce3ee290..77f8395ac898e 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,9 @@ buildscript { ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x ext.jacksonVersion = '2.15.3' ext.jettyVersion = '11.0.21' + // see also datahub-frontend/play.gradle ext.playVersion = '2.8.22' + ext.playScalaVersion = '2.13' ext.log4jVersion = '2.23.1' ext.slf4jVersion = '1.7.36' ext.logbackClassic = '1.4.14' @@ -103,7 +105,7 @@ project.ext.spec = [ ] project.ext.externalDependency = [ - 'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10', + 'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3', 'antlr4': 'org.antlr:antlr4:4.9.3', 'assertJ': 'org.assertj:assertj-core:3.11.1', @@ -212,18 +214,18 @@ project.ext.externalDependency = [ 'parquet': 'org.apache.parquet:parquet-avro:1.12.3', 'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1', 'picocli': 'info.picocli:picocli:4.5.0', - 'playCache': "com.typesafe.play:play-cache_2.12:$playVersion", - 'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion", - 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10', - 'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion", - 'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion", - 'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion", - 'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion", - 'playServer': "com.typesafe.play:play-server_2.12:$playVersion", - 'playTest': "com.typesafe.play:play-test_2.12:$playVersion", - 'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion", - 'pac4j': 'org.pac4j:pac4j-oidc:4.5.8', - 'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2', + 'playCache': "com.typesafe.play:play-cache_$playScalaVersion:$playVersion", + 'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_$playScalaVersion:$playVersion", + 'playWs': "com.typesafe.play:play-ahc-ws-standalone_$playScalaVersion:2.1.10", + 'playDocs': "com.typesafe.play:play-docs_$playScalaVersion:$playVersion", + 'playGuice': "com.typesafe.play:play-guice_$playScalaVersion:$playVersion", + 'playJavaJdbc': "com.typesafe.play:play-java-jdbc_$playScalaVersion:$playVersion", + 'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_$playScalaVersion:$playVersion", + 'playServer': "com.typesafe.play:play-server_$playScalaVersion:$playVersion", + 'playTest': "com.typesafe.play:play-test_$playScalaVersion:$playVersion", + 'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion", + 'pac4j': 'org.pac4j:pac4j-oidc:6.0.6', + 'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8", 'postgresql': 'org.postgresql:postgresql:42.3.9', 'protobuf': 'com.google.protobuf:protobuf-java:3.25.5', 'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0', @@ -407,6 +409,8 @@ subprojects { googleJavaFormat() target project.fileTree(project.projectDir) { include 'src/**/*.java' + include 'app/**/*.java' + include 'test/**/*.java' exclude 'src/**/resources/' exclude 'src/**/generated/' exclude 'src/**/mainGeneratedDataTemplate/' diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 7bb2547890701..d51795330f5ce 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -21,27 +21,28 @@ import com.linkedin.util.Configuration; import config.ConfigurationProvider; import controllers.SsoCallbackController; -import io.datahubproject.metadata.context.ValidationContext; -import java.nio.charset.StandardCharsets; -import java.util.Collections; - import io.datahubproject.metadata.context.ActorContext; import io.datahubproject.metadata.context.AuthorizationContext; import io.datahubproject.metadata.context.EntityRegistryContext; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.OperationContextConfig; import io.datahubproject.metadata.context.SearchContext; +import io.datahubproject.metadata.context.ValidationContext; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.pac4j.core.config.Config; import org.pac4j.core.context.session.SessionStore; +import org.pac4j.core.profile.ProfileManager; +import org.pac4j.core.util.serializer.JavaSerializer; import org.pac4j.play.LogoutController; import org.pac4j.play.http.PlayHttpActionAdapter; import org.pac4j.play.store.PlayCacheSessionStore; import org.pac4j.play.store.PlayCookieSessionStore; -import org.pac4j.play.store.PlaySessionStore; import org.pac4j.play.store.ShiroAesDataEncrypter; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import play.Environment; @@ -63,14 +64,16 @@ public class AuthModule extends AbstractModule { private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider"; private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval"; private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries"; - private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize"; - private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency"; + private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = + "entityClient.restli.get.batchSize"; + private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = + "entityClient.restli.get.batchConcurrency"; private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings"; - private final com.typesafe.config.Config _configs; + private final com.typesafe.config.Config configs; public AuthModule(final Environment environment, final com.typesafe.config.Config configs) { - _configs = configs; + this.configs = configs; } @Override @@ -84,13 +87,13 @@ protected void configure() { * the response will be rejected by the browser. Default to PlayCacheCookieStore so that * datahub-frontend container remains as a stateless service */ - String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF); + String sessionStoreProvider = configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF); if (sessionStoreProvider.equals("PlayCacheSessionStore")) { final PlayCacheSessionStore playCacheSessionStore = new PlayCacheSessionStore(getProvider(SyncCacheApi.class)); bind(SessionStore.class).toInstance(playCacheSessionStore); - bind(PlaySessionStore.class).toInstance(playCacheSessionStore); + bind(PlayCacheSessionStore.class).toInstance(playCacheSessionStore); } else { PlayCookieSessionStore playCacheCookieStore; try { @@ -98,17 +101,18 @@ protected void configure() { // hash the input to generate a fixed-length string. Then, we convert // it to hex and slice the first 16 bytes, because AES key length must strictly // have a specific length. - final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF); + final String aesKeyBase = configs.getString(PAC4J_AES_KEY_BASE_CONF); final String aesKeyHash = DigestUtils.sha256Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8)); final String aesEncryptionKey = aesKeyHash.substring(0, 16); playCacheCookieStore = new PlayCookieSessionStore(new ShiroAesDataEncrypter(aesEncryptionKey.getBytes())); + playCacheCookieStore.setSerializer(new JavaSerializer()); } catch (Exception e) { throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e); } bind(SessionStore.class).toInstance(playCacheCookieStore); - bind(PlaySessionStore.class).toInstance(playCacheCookieStore); + bind(PlayCookieSessionStore.class).toInstance(playCacheCookieStore); } try { @@ -133,9 +137,12 @@ protected void configure() { @Provides @Singleton - protected Config provideConfig() { + protected Config provideConfig(@Nonnull SessionStore sessionStore) { Config config = new Config(); + config.setSessionStoreFactory(parameters -> sessionStore); config.setHttpActionAdapter(new PlayHttpActionAdapter()); + config.setProfileManagerFactory(ProfileManager::new); + return config; } @@ -145,7 +152,7 @@ protected SsoManager provideSsoManager( Authentication systemAuthentication, CloseableHttpClient httpClient) { SsoManager manager = new SsoManager( - _configs, systemAuthentication, getSsoSettingsRequestUrl(_configs), httpClient); + configs, systemAuthentication, getSsoSettingsRequestUrl(configs), httpClient); manager.initializeSsoProvider(); return manager; } @@ -155,8 +162,8 @@ protected SsoManager provideSsoManager( protected Authentication provideSystemAuthentication() { // Returns an instance of Authentication used to authenticate system initiated calls to Metadata // Service. - String systemClientId = _configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH); - String systemSecret = _configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH); + String systemClientId = configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH); + String systemSecret = configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH); final Actor systemActor = new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported. return new Authentication( @@ -169,27 +176,25 @@ protected Authentication provideSystemAuthentication() { @Singleton @Named("systemOperationContext") protected OperationContext provideOperationContext( - final Authentication systemAuthentication, - final ConfigurationProvider configurationProvider) { + final Authentication systemAuthentication, + final ConfigurationProvider configurationProvider) { ActorContext systemActorContext = - ActorContext.builder() - .systemAuth(true) - .authentication(systemAuthentication) - .build(); - OperationContextConfig systemConfig = OperationContextConfig.builder() + ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build(); + OperationContextConfig systemConfig = + OperationContextConfig.builder() .viewAuthorizationConfiguration(configurationProvider.getAuthorization().getView()) .allowSystemAuthentication(true) .build(); return OperationContext.builder() - .operationContextConfig(systemConfig) - .systemActorContext(systemActorContext) - // Authorizer.EMPTY is fine since it doesn't actually apply to system auth - .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build()) - .searchContext(SearchContext.EMPTY) - .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY)) - .validationContext(ValidationContext.builder().alternateValidation(false).build()) - .build(systemAuthentication); + .operationContextConfig(systemConfig) + .systemActorContext(systemActorContext) + // Authorizer.EMPTY is fine since it doesn't actually apply to system auth + .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build()) + .searchContext(SearchContext.EMPTY) + .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY)) + .validationContext(ValidationContext.builder().alternateValidation(false).build()) + .build(systemAuthentication); } @Provides @@ -208,11 +213,11 @@ protected SystemEntityClient provideEntityClient( return new SystemRestliEntityClient( buildRestliClient(), - new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)), - _configs.getInt(ENTITY_CLIENT_NUM_RETRIES), + new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)), + configs.getInt(ENTITY_CLIENT_NUM_RETRIES), configurationProvider.getCache().getClient().getEntityClient(), - Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)), - Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY))); + Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)), + Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY))); } @Provides @@ -220,11 +225,11 @@ protected SystemEntityClient provideEntityClient( protected AuthServiceClient provideAuthClient( Authentication systemAuthentication, CloseableHttpClient httpClient) { // Init a GMS auth client - final String metadataServiceHost = getMetadataServiceHost(_configs); + final String metadataServiceHost = getMetadataServiceHost(configs); - final int metadataServicePort = getMetadataServicePort(_configs); + final int metadataServicePort = getMetadataServicePort(configs); - final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(_configs); + final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(configs); return new AuthServiceClient( metadataServiceHost, @@ -243,22 +248,22 @@ protected CloseableHttpClient provideHttpClient() { private com.linkedin.restli.client.Client buildRestliClient() { final String metadataServiceHost = utils.ConfigUtil.getString( - _configs, + configs, METADATA_SERVICE_HOST_CONFIG_PATH, utils.ConfigUtil.DEFAULT_METADATA_SERVICE_HOST); final int metadataServicePort = utils.ConfigUtil.getInt( - _configs, + configs, utils.ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH, utils.ConfigUtil.DEFAULT_METADATA_SERVICE_PORT); final boolean metadataServiceUseSsl = utils.ConfigUtil.getBoolean( - _configs, + configs, utils.ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH, ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL); final String metadataServiceSslProtocol = utils.ConfigUtil.getString( - _configs, + configs, utils.ConfigUtil.METADATA_SERVICE_SSL_PROTOCOL_CONFIG_PATH, ConfigUtil.DEFAULT_METADATA_SERVICE_SSL_PROTOCOL); return DefaultRestliClientFactory.getRestLiClient( diff --git a/datahub-frontend/app/auth/AuthUtils.java b/datahub-frontend/app/auth/AuthUtils.java index 51bb784c61b3b..490f52bece651 100644 --- a/datahub-frontend/app/auth/AuthUtils.java +++ b/datahub-frontend/app/auth/AuthUtils.java @@ -75,6 +75,7 @@ public class AuthUtils { public static final String RESPONSE_MODE = "responseMode"; public static final String USE_NONCE = "useNonce"; public static final String READ_TIMEOUT = "readTimeout"; + public static final String CONNECT_TIMEOUT = "connectTimeout"; public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims"; // Retained for backwards compatibility public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm"; diff --git a/datahub-frontend/app/auth/CookieConfigs.java b/datahub-frontend/app/auth/CookieConfigs.java index 63b2ce61aaf9b..e77e200144835 100644 --- a/datahub-frontend/app/auth/CookieConfigs.java +++ b/datahub-frontend/app/auth/CookieConfigs.java @@ -10,34 +10,34 @@ public class CookieConfigs { public static final String AUTH_COOKIE_SECURE = "play.http.session.secure"; public static final boolean DEFAULT_AUTH_COOKIE_SECURE = false; - private final int _ttlInHours; - private final String _authCookieSameSite; - private final boolean _authCookieSecure; + private final int ttlInHours; + private final String authCookieSameSite; + private final boolean authCookieSecure; public CookieConfigs(final Config configs) { - _ttlInHours = + ttlInHours = configs.hasPath(SESSION_TTL_CONFIG_PATH) ? configs.getInt(SESSION_TTL_CONFIG_PATH) : DEFAULT_SESSION_TTL_HOURS; - _authCookieSameSite = + authCookieSameSite = configs.hasPath(AUTH_COOKIE_SAME_SITE) ? configs.getString(AUTH_COOKIE_SAME_SITE) : DEFAULT_AUTH_COOKIE_SAME_SITE; - _authCookieSecure = + authCookieSecure = configs.hasPath(AUTH_COOKIE_SECURE) ? configs.getBoolean(AUTH_COOKIE_SECURE) : DEFAULT_AUTH_COOKIE_SECURE; } public int getTtlInHours() { - return _ttlInHours; + return ttlInHours; } public String getAuthCookieSameSite() { - return _authCookieSameSite; + return authCookieSameSite; } public boolean getAuthCookieSecure() { - return _authCookieSecure; + return authCookieSecure; } } diff --git a/datahub-frontend/app/auth/JAASConfigs.java b/datahub-frontend/app/auth/JAASConfigs.java index 529bf98e1fdcf..dee4ded68808a 100644 --- a/datahub-frontend/app/auth/JAASConfigs.java +++ b/datahub-frontend/app/auth/JAASConfigs.java @@ -8,16 +8,16 @@ public class JAASConfigs { public static final String JAAS_ENABLED_CONFIG_PATH = "auth.jaas.enabled"; - private Boolean _isEnabled = true; + private Boolean isEnabled = true; public JAASConfigs(final com.typesafe.config.Config configs) { if (configs.hasPath(JAAS_ENABLED_CONFIG_PATH) && !configs.getBoolean(JAAS_ENABLED_CONFIG_PATH)) { - _isEnabled = false; + isEnabled = false; } } public boolean isJAASEnabled() { - return _isEnabled; + return isEnabled; } } diff --git a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java index 772c2c8f92f28..a7b8a8bc80067 100644 --- a/datahub-frontend/app/auth/NativeAuthenticationConfigs.java +++ b/datahub-frontend/app/auth/NativeAuthenticationConfigs.java @@ -7,17 +7,17 @@ public class NativeAuthenticationConfigs { public static final String NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH = "auth.native.signUp.enforceValidEmail"; - private Boolean _isEnabled = true; - private Boolean _isEnforceValidEmailEnabled = true; + private Boolean isEnabled = true; + private Boolean isEnforceValidEmailEnabled = true; public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) { if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)) { - _isEnabled = + isEnabled = Boolean.parseBoolean( configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString()); } if (configs.hasPath(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)) { - _isEnforceValidEmailEnabled = + isEnforceValidEmailEnabled = Boolean.parseBoolean( configs .getValue(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH) @@ -26,10 +26,10 @@ public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) { } public boolean isNativeAuthenticationEnabled() { - return _isEnabled; + return isEnabled; } public boolean isEnforceValidEmailEnabled() { - return _isEnforceValidEmailEnabled; + return isEnforceValidEmailEnabled; } } diff --git a/datahub-frontend/app/auth/sso/SsoConfigs.java b/datahub-frontend/app/auth/sso/SsoConfigs.java index 976d0826f2277..46a2b7bfd27e8 100644 --- a/datahub-frontend/app/auth/sso/SsoConfigs.java +++ b/datahub-frontend/app/auth/sso/SsoConfigs.java @@ -1,10 +1,9 @@ package auth.sso; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - import static auth.AuthUtils.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; /** * Class responsible for extracting and validating top-level SSO related configurations. TODO: @@ -25,72 +24,72 @@ public class SsoConfigs { private static final String DEFAULT_SUCCESS_REDIRECT_PATH = "/"; - private final String _authBaseUrl; - private final String _authBaseCallbackPath; - private final String _authSuccessRedirectPath; - private final Boolean _oidcEnabled; + private final String authBaseUrl; + private final String authBaseCallbackPath; + private final String authSuccessRedirectPath; + private final Boolean oidcEnabled; public SsoConfigs(Builder builder) { - _authBaseUrl = builder._authBaseUrl; - _authBaseCallbackPath = builder._authBaseCallbackPath; - _authSuccessRedirectPath = builder._authSuccessRedirectPath; - _oidcEnabled = builder._oidcEnabled; + authBaseUrl = builder.authBaseUrl; + authBaseCallbackPath = builder.authBaseCallbackPath; + authSuccessRedirectPath = builder.authSuccessRedirectPath; + oidcEnabled = builder.oidcEnabled; } public String getAuthBaseUrl() { - return _authBaseUrl; + return authBaseUrl; } public String getAuthBaseCallbackPath() { - return _authBaseCallbackPath; + return authBaseCallbackPath; } public String getAuthSuccessRedirectPath() { - return _authSuccessRedirectPath; + return authSuccessRedirectPath; } public Boolean isOidcEnabled() { - return _oidcEnabled; + return oidcEnabled; } public static class Builder> { - protected String _authBaseUrl = null; - private String _authBaseCallbackPath = DEFAULT_BASE_CALLBACK_PATH; - private String _authSuccessRedirectPath = DEFAULT_SUCCESS_REDIRECT_PATH; - protected Boolean _oidcEnabled = false; - private final ObjectMapper _objectMapper = new ObjectMapper(); + protected String authBaseUrl = null; + private String authBaseCallbackPath = DEFAULT_BASE_CALLBACK_PATH; + private String authSuccessRedirectPath = DEFAULT_SUCCESS_REDIRECT_PATH; + protected Boolean oidcEnabled = false; + private final ObjectMapper objectMapper = new ObjectMapper(); protected JsonNode jsonNode = null; // No need to check if changes are made since this method is only called at start-up. public Builder from(final com.typesafe.config.Config configs) { if (configs.hasPath(AUTH_BASE_URL_CONFIG_PATH)) { - _authBaseUrl = configs.getString(AUTH_BASE_URL_CONFIG_PATH); + authBaseUrl = configs.getString(AUTH_BASE_URL_CONFIG_PATH); } if (configs.hasPath(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH)) { - _authBaseCallbackPath = configs.getString(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH); + authBaseCallbackPath = configs.getString(AUTH_BASE_CALLBACK_PATH_CONFIG_PATH); } if (configs.hasPath(OIDC_ENABLED_CONFIG_PATH)) { - _oidcEnabled = + oidcEnabled = Boolean.TRUE.equals(Boolean.parseBoolean(configs.getString(OIDC_ENABLED_CONFIG_PATH))); } if (configs.hasPath(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH)) { - _authSuccessRedirectPath = configs.getString(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH); + authSuccessRedirectPath = configs.getString(AUTH_SUCCESS_REDIRECT_PATH_CONFIG_PATH); } return this; } public Builder from(String ssoSettingsJsonStr) { try { - jsonNode = _objectMapper.readTree(ssoSettingsJsonStr); + jsonNode = objectMapper.readTree(ssoSettingsJsonStr); } catch (Exception e) { throw new RuntimeException( String.format("Failed to parse ssoSettingsJsonStr %s into JSON", ssoSettingsJsonStr)); } if (jsonNode.has(BASE_URL)) { - _authBaseUrl = jsonNode.get(BASE_URL).asText(); + authBaseUrl = jsonNode.get(BASE_URL).asText(); } if (jsonNode.has(OIDC_ENABLED)) { - _oidcEnabled = jsonNode.get(OIDC_ENABLED).asBoolean(); + oidcEnabled = jsonNode.get(OIDC_ENABLED).asBoolean(); } return this; diff --git a/datahub-frontend/app/auth/sso/SsoManager.java b/datahub-frontend/app/auth/sso/SsoManager.java index 8377eb40e237f..8a8a7f95a1b60 100644 --- a/datahub-frontend/app/auth/sso/SsoManager.java +++ b/datahub-frontend/app/auth/sso/SsoManager.java @@ -26,22 +26,21 @@ public class SsoManager { private SsoProvider _provider; // Only one active provider at a time. - private final Authentication - _authentication; // Authentication used to fetch SSO settings from GMS - private final String _ssoSettingsRequestUrl; // SSO settings request URL. - private final CloseableHttpClient _httpClient; // HTTP client for making requests to GMS. - private com.typesafe.config.Config _configs; + private final Authentication authentication; // Authentication used to fetch SSO settings from GMS + private final String ssoSettingsRequestUrl; // SSO settings request URL. + private final CloseableHttpClient httpClient; // HTTP client for making requests to GMS. + private com.typesafe.config.Config configs; public SsoManager( com.typesafe.config.Config configs, Authentication authentication, String ssoSettingsRequestUrl, CloseableHttpClient httpClient) { - _configs = configs; - _authentication = Objects.requireNonNull(authentication, "authentication cannot be null"); - _ssoSettingsRequestUrl = + this.configs = configs; + this.authentication = Objects.requireNonNull(authentication, "authentication cannot be null"); + this.ssoSettingsRequestUrl = Objects.requireNonNull(ssoSettingsRequestUrl, "ssoSettingsRequestUrl cannot be null"); - _httpClient = Objects.requireNonNull(httpClient, "httpClient cannot be null"); + this.httpClient = Objects.requireNonNull(httpClient, "httpClient cannot be null"); _provider = null; } @@ -52,6 +51,9 @@ public SsoManager( * @return true if SSO logic is enabled, false otherwise. */ public boolean isSsoEnabled() { + if (configs.hasPath("auth.oidc.enabled") && configs.getBoolean("auth.oidc.enabled")) { + return true; + } refreshSsoProvider(); return _provider != null; } @@ -66,7 +68,7 @@ public void setSsoProvider(final SsoProvider provider) { } public void setConfigs(final com.typesafe.config.Config configs) { - _configs = configs; + this.configs = configs; } public void clearSsoProvider() { @@ -87,19 +89,19 @@ public SsoProvider getSsoProvider() { public void initializeSsoProvider() { SsoConfigs ssoConfigs = null; try { - ssoConfigs = new SsoConfigs.Builder().from(_configs).build(); + ssoConfigs = new SsoConfigs.Builder().from(configs).build(); } catch (Exception e) { // Debug-level logging since this is expected to fail if SSO has not been configured. - log.debug(String.format("Missing SSO settings in static configs %s", _configs), e); + log.debug(String.format("Missing SSO settings in static configs %s", configs), e); } if (ssoConfigs != null && ssoConfigs.isOidcEnabled()) { try { - OidcConfigs oidcConfigs = new OidcConfigs.Builder().from(_configs).build(); + OidcConfigs oidcConfigs = new OidcConfigs.Builder().from(configs).build(); maybeUpdateOidcProvider(oidcConfigs); } catch (Exception e) { // Error-level logging since this is unexpected to fail if SSO has been configured. - log.error(String.format("Error building OidcConfigs from static configs %s", _configs), e); + log.error(String.format("Error building OidcConfigs from static configs %s", configs), e); } } else { // Clear the SSO Provider since no SSO is enabled. @@ -132,7 +134,7 @@ private void refreshSsoProvider() { if (ssoConfigs != null && ssoConfigs.isOidcEnabled()) { try { OidcConfigs oidcConfigs = - new OidcConfigs.Builder().from(_configs, ssoSettingsJsonStr).build(); + new OidcConfigs.Builder().from(configs, ssoSettingsJsonStr).build(); maybeUpdateOidcProvider(oidcConfigs); } catch (Exception e) { log.error( @@ -166,15 +168,15 @@ private void maybeUpdateOidcProvider(OidcConfigs oidcConfigs) { private Optional getDynamicSsoSettings() { CloseableHttpResponse response = null; try { - final HttpPost request = new HttpPost(_ssoSettingsRequestUrl); + final HttpPost request = new HttpPost(ssoSettingsRequestUrl); // Build JSON request to verify credentials for a native user. request.setEntity(new StringEntity("")); // Add authorization header with DataHub frontend system id and secret. - request.addHeader(Http.HeaderNames.AUTHORIZATION, _authentication.getCredentials()); + request.addHeader(Http.HeaderNames.AUTHORIZATION, authentication.getCredentials()); - response = _httpClient.execute(request); + response = httpClient.execute(request); final HttpEntity entity = response.getEntity(); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && entity != null) { // Successfully received the SSO settings diff --git a/datahub-frontend/app/auth/sso/SsoProvider.java b/datahub-frontend/app/auth/sso/SsoProvider.java index a0947b52b92ae..2a6fff728966e 100644 --- a/datahub-frontend/app/auth/sso/SsoProvider.java +++ b/datahub-frontend/app/auth/sso/SsoProvider.java @@ -1,7 +1,6 @@ package auth.sso; import org.pac4j.core.client.Client; -import org.pac4j.core.credentials.Credentials; /** A thin interface over a Pac4j {@link Client} object and its associated configurations. */ public interface SsoProvider { @@ -12,14 +11,14 @@ enum SsoProtocol { // SAML -- not yet supported. // Common name appears in the Callback URL itself. - private final String _commonName; + private final String commonName; public String getCommonName() { - return _commonName; + return commonName; } SsoProtocol(String commonName) { - _commonName = commonName; + this.commonName = commonName; } } @@ -30,5 +29,5 @@ public String getCommonName() { SsoProtocol protocol(); /** Retrieves an initialized Pac4j {@link Client}. */ - Client client(); + Client client(); } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java index fa676d2d16c90..3a4433b0ca81e 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcAuthorizationGenerator.java @@ -5,7 +5,7 @@ import java.util.Map.Entry; import java.util.Optional; import org.pac4j.core.authorization.generator.AuthorizationGenerator; -import org.pac4j.core.context.WebContext; +import org.pac4j.core.context.CallContext; import org.pac4j.core.profile.AttributeLocation; import org.pac4j.core.profile.CommonProfile; import org.pac4j.core.profile.UserProfile; @@ -18,24 +18,31 @@ public class OidcAuthorizationGenerator implements AuthorizationGenerator { private static final Logger logger = LoggerFactory.getLogger(OidcAuthorizationGenerator.class); - private final ProfileDefinition profileDef; - + private final ProfileDefinition profileDef; private final OidcConfigs oidcConfigs; public OidcAuthorizationGenerator( - final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) { + final ProfileDefinition profileDef, final OidcConfigs oidcConfigs) { this.profileDef = profileDef; this.oidcConfigs = oidcConfigs; } @Override - public Optional generate(WebContext context, UserProfile profile) { + public Optional generate(final CallContext context, final UserProfile profile) { + if (!(profile instanceof OidcProfile oidcProfile)) { + return Optional.of(profile); + } + if (oidcConfigs.getExtractJwtAccessTokenClaims().orElse(false)) { try { - final JWT jwt = JWTParser.parse(((OidcProfile) profile).getAccessToken().getValue()); + final JWT jwt = JWTParser.parse(oidcProfile.getAccessToken().getValue()); CommonProfile commonProfile = new CommonProfile(); + // Copy existing attributes + profile.getAttributes().forEach(commonProfile::addAttribute); + + // Add JWT claims for (final Entry entry : jwt.getJWTClaimsSet().getClaims().entrySet()) { final String claimName = entry.getKey(); @@ -51,6 +58,6 @@ public Optional generate(WebContext context, UserProfile profile) { } } - return Optional.ofNullable(profile); + return Optional.of(profile); } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index 510804ba17f1a..ef5833f607efd 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -39,6 +39,8 @@ import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.util.Pair; +import io.datahubproject.metadata.context.OperationContext; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URI; @@ -51,27 +53,36 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; - -import io.datahubproject.metadata.context.OperationContext; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; +import org.pac4j.core.client.BaseClient; +import org.pac4j.core.client.Client; +import org.pac4j.core.client.Clients; import org.pac4j.core.config.Config; +import org.pac4j.core.context.CallContext; import org.pac4j.core.context.Cookie; +import org.pac4j.core.context.FrameworkParameters; +import org.pac4j.core.context.WebContext; +import org.pac4j.core.credentials.Credentials; import org.pac4j.core.engine.DefaultCallbackLogic; +import org.pac4j.core.exception.http.HttpAction; import org.pac4j.core.http.adapter.HttpActionAdapter; import org.pac4j.core.profile.CommonProfile; import org.pac4j.core.profile.ProfileManager; import org.pac4j.core.profile.UserProfile; +import org.pac4j.core.util.CommonHelper; import org.pac4j.core.util.Pac4jConstants; -import org.pac4j.play.PlayWebContext; +import org.pac4j.play.store.PlayCookieSessionStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import play.mvc.Result; -import javax.annotation.Nonnull; - /** * This class contains the logic that is executed when an OpenID Connect Identity Provider redirects * back to D DataHub after an authentication attempt. @@ -83,7 +94,8 @@ * if the user does not already exist. */ @Slf4j -public class OidcCallbackLogic extends DefaultCallbackLogic { +public class OidcCallbackLogic extends DefaultCallbackLogic { + private static final Logger LOGGER = LoggerFactory.getLogger(OidcCallbackLogic.class); private final SsoManager ssoManager; private final SystemEntityClient systemEntityClient; @@ -105,67 +117,129 @@ public OidcCallbackLogic( } @Override - public Result perform( - PlayWebContext context, + public Object perform( Config config, - HttpActionAdapter httpActionAdapter, - String defaultUrl, - Boolean saveInSession, - Boolean multiProfile, - Boolean renewSession, - String defaultClient) { - - setContextRedirectUrl(context); - - final Result result = - super.perform( - context, - config, - httpActionAdapter, - defaultUrl, - saveInSession, - multiProfile, - renewSession, - defaultClient); + String inputDefaultUrl, + Boolean inputRenewSession, + String defaultClient, + FrameworkParameters parameters) { + + final Pair ctxResult = + superPerform(config, inputDefaultUrl, inputRenewSession, defaultClient, parameters); + + CallContext ctx = ctxResult.getFirst(); + Result result = (Result) ctxResult.getSecond(); + + setContextRedirectUrl(ctx); // Handle OIDC authentication errors. - if (OidcResponseErrorHandler.isError(context)) { - return OidcResponseErrorHandler.handleError(context); + if (OidcResponseErrorHandler.isError(ctx)) { + return OidcResponseErrorHandler.handleError(ctx); } // By this point, we know that OIDC is the enabled provider. final OidcConfigs oidcConfigs = (OidcConfigs) ssoManager.getSsoProvider().configs(); - return handleOidcCallback(systemOperationContext, oidcConfigs, result, getProfileManager(context)); + return handleOidcCallback(systemOperationContext, ctx, oidcConfigs, result); + } + + /** Overriding this to be able to intercept the CallContext being created */ + private Pair superPerform( + Config config, + String inputDefaultUrl, + Boolean inputRenewSession, + String defaultClient, + FrameworkParameters parameters) { + LOGGER.debug("=== CALLBACK ==="); + CallContext ctx = this.buildContext(config, parameters); + WebContext webContext = ctx.webContext(); + HttpActionAdapter httpActionAdapter = config.getHttpActionAdapter(); + CommonHelper.assertNotNull("httpActionAdapter", httpActionAdapter); + + HttpAction action; + try { + CommonHelper.assertNotNull("clientFinder", getClientFinder()); + String defaultUrl = (String) Objects.requireNonNullElse(inputDefaultUrl, "/"); + boolean renewSession = inputRenewSession == null || inputRenewSession; + CommonHelper.assertNotBlank("defaultUrl", defaultUrl); + Clients clients = config.getClients(); + CommonHelper.assertNotNull("clients", clients); + List foundClients = getClientFinder().find(clients, webContext, defaultClient); + CommonHelper.assertTrue( + foundClients != null && foundClients.size() == 1, + "unable to find one indirect client for the callback: check the callback URL for a client name parameter or suffix path or ensure that your configuration defaults to one indirect client"); + Client foundClient = (Client) foundClients.get(0); + LOGGER.debug("foundClient: {}", foundClient); + CommonHelper.assertNotNull("foundClient", foundClient); + Credentials credentials = (Credentials) foundClient.getCredentials(ctx).orElse(null); + LOGGER.debug("extracted credentials: {}", credentials); + credentials = (Credentials) foundClient.validateCredentials(ctx, credentials).orElse(null); + LOGGER.debug("validated credentials: {}", credentials); + if (credentials != null && !credentials.isForAuthentication()) { + action = foundClient.processLogout(ctx, credentials); + } else { + if (credentials != null) { + Optional optProfile = foundClient.getUserProfile(ctx, credentials); + LOGGER.debug("optProfile: {}", optProfile); + if (optProfile.isPresent()) { + UserProfile profile = (UserProfile) optProfile.get(); + Boolean saveProfileInSession = + ((BaseClient) foundClient).getSaveProfileInSession(webContext, profile); + boolean multiProfile = ((BaseClient) foundClient).isMultiProfile(webContext, profile); + LOGGER.debug( + "saveProfileInSession: {} / multiProfile: {}", saveProfileInSession, multiProfile); + this.saveUserProfile( + ctx, config, profile, saveProfileInSession, multiProfile, renewSession); + } + } + + action = this.redirectToOriginallyRequestedUrl(ctx, defaultUrl); + } + } catch (RuntimeException var20) { + RuntimeException e = var20; + return Pair.of(ctx, this.handleException(e, httpActionAdapter, webContext)); + } + + return Pair.of(ctx, httpActionAdapter.adapt(action, webContext)); } - @SuppressWarnings("unchecked") - private void setContextRedirectUrl(PlayWebContext context) { + private void setContextRedirectUrl(CallContext ctx) { + WebContext context = ctx.webContext(); + PlayCookieSessionStore sessionStore = (PlayCookieSessionStore) ctx.sessionStore(); + Optional redirectUrl = context.getRequestCookies().stream() .filter(cookie -> REDIRECT_URL_COOKIE_NAME.equals(cookie.getName())) .findFirst(); redirectUrl.ifPresent( cookie -> - context - .getSessionStore() - .set( - context, - Pac4jConstants.REQUESTED_URL, - JAVA_SER_HELPER.deserializeFromBytes( + sessionStore.set( + context, + Pac4jConstants.REQUESTED_URL, + sessionStore + .getSerializer() + .deserializeFromBytes( uncompressBytes(Base64.getDecoder().decode(cookie.getValue()))))); } private Result handleOidcCallback( final OperationContext opContext, + final CallContext ctx, final OidcConfigs oidcConfigs, - final Result result, - final ProfileManager profileManager) { + final Result result) { log.debug("Beginning OIDC Callback Handling..."); + ProfileManager profileManager = + ctx.profileManagerFactory().apply(ctx.webContext(), ctx.sessionStore()); + if (profileManager.isAuthenticated()) { // If authenticated, the user should have a profile. - final CommonProfile profile = (CommonProfile) profileManager.get(true).get(); + final Optional optProfile = profileManager.getProfile(); + if (optProfile.isEmpty()) { + return internalServerError( + "Failed to authenticate current user. Cannot find valid identity provider profile in session."); + } + final CommonProfile profile = (CommonProfile) optProfile.get(); log.debug( String.format( "Found authenticated user with profile %s", profile.getAttributes().toString())); @@ -196,7 +270,8 @@ private Result handleOidcCallback( } // Update user status to active on login. // If we want to prevent certain users from logging in, here's where we'll want to do it. - setUserStatus(opContext, + setUserStatus( + opContext, corpUserUrn, new CorpUserStatus() .setStatus(Constants.CORP_USER_STATUS_ACTIVE) @@ -307,29 +382,32 @@ private CorpUserSnapshot extractUser(CorpuserUrn urn, CommonProfile profile) { return corpUserSnapshot; } - public static Collection getGroupNames(CommonProfile profile, Object groupAttribute, String groupsClaimName) { - Collection groupNames = Collections.emptyList(); - try { - if (groupAttribute instanceof Collection) { - // List of group names - groupNames = (Collection) profile.getAttribute(groupsClaimName, Collection.class); - } else if (groupAttribute instanceof String) { - String groupString = (String) groupAttribute; - ObjectMapper objectMapper = new ObjectMapper(); - try { - // Json list of group names - groupNames = objectMapper.readValue(groupString, new TypeReference>(){}); - } catch (Exception e) { - groupNames = Arrays.asList(groupString.split(",")); - } + public static Collection getGroupNames( + CommonProfile profile, Object groupAttribute, String groupsClaimName) { + Collection groupNames = Collections.emptyList(); + try { + if (groupAttribute instanceof Collection) { + // List of group names + groupNames = (Collection) profile.getAttribute(groupsClaimName, Collection.class); + } else if (groupAttribute instanceof String) { + String groupString = (String) groupAttribute; + ObjectMapper objectMapper = new ObjectMapper(); + try { + // Json list of group names + groupNames = objectMapper.readValue(groupString, new TypeReference>() {}); + } catch (Exception e) { + groupNames = Arrays.asList(groupString.split(",")); } - } catch (Exception e) { - log.error(String.format( - "Failed to parse group names: Expected to find a list of strings for attribute with name %s, found %s", - groupsClaimName, profile.getAttribute(groupsClaimName).getClass())); } - return groupNames; + } catch (Exception e) { + log.error( + String.format( + "Failed to parse group names: Expected to find a list of strings for attribute with name %s, found %s", + groupsClaimName, profile.getAttribute(groupsClaimName).getClass())); + } + return groupNames; } + private List extractGroups(CommonProfile profile) { log.debug( @@ -350,7 +428,8 @@ private List extractGroups(CommonProfile profile) { if (profile.containsAttribute(groupsClaimName)) { try { final List groupSnapshots = new ArrayList<>(); - Collection groupNames = getGroupNames(profile, profile.getAttribute(groupsClaimName), groupsClaimName); + Collection groupNames = + getGroupNames(profile, profile.getAttribute(groupsClaimName), groupsClaimName); for (String groupName : groupNames) { // Create a basic CorpGroupSnapshot from the information. @@ -405,7 +484,8 @@ private GroupMembership createGroupMembership(final List extr return groupMembershipAspect; } - private void tryProvisionUser(@Nonnull OperationContext opContext, CorpUserSnapshot corpUserSnapshot) { + private void tryProvisionUser( + @Nonnull OperationContext opContext, CorpUserSnapshot corpUserSnapshot) { log.debug(String.format("Attempting to provision user with urn %s", corpUserSnapshot.getUrn())); @@ -439,7 +519,8 @@ private void tryProvisionUser(@Nonnull OperationContext opContext, CorpUserSnaps } } - private void tryProvisionGroups(@Nonnull OperationContext opContext, List corpGroups) { + private void tryProvisionGroups( + @Nonnull OperationContext opContext, List corpGroups) { log.debug( String.format( @@ -450,8 +531,7 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List urnsToFetch = corpGroups.stream().map(CorpGroupSnapshot::getUrn).collect(Collectors.toSet()); - final Map existingGroups = - systemEntityClient.batchGet(opContext, urnsToFetch); + final Map existingGroups = systemEntityClient.batchGet(opContext, urnsToFetch); log.debug(String.format("Fetched GMS groups with urns %s", existingGroups.keySet())); @@ -489,7 +569,8 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List new Entity().setValue(Snapshot.create(groupSnapshot))) .collect(Collectors.toSet())); @@ -505,7 +586,8 @@ private void tryProvisionGroups(@Nonnull OperationContext opContext, List useNonce; private final Optional customParamResource; private final String readTimeout; + private final String connectTimeout; private final Optional extractJwtAccessTokenClaims; private final Optional preferredJwsAlgorithm; private final Optional grantType; @@ -100,6 +103,7 @@ public OidcConfigs(Builder builder) { this.useNonce = builder.useNonce; this.customParamResource = builder.customParamResource; this.readTimeout = builder.readTimeout; + this.connectTimeout = builder.connectTimeout; this.extractJwtAccessTokenClaims = builder.extractJwtAccessTokenClaims; this.preferredJwsAlgorithm = builder.preferredJwsAlgorithm; this.acrValues = builder.acrValues; @@ -127,6 +131,7 @@ public static class Builder extends SsoConfigs.Builder { private Optional useNonce = Optional.empty(); private Optional customParamResource = Optional.empty(); private String readTimeout = DEFAULT_OIDC_READ_TIMEOUT; + private String connectTimeout = DEFAULT_OIDC_CONNECT_TIMEOUT; private Optional extractJwtAccessTokenClaims = Optional.empty(); private Optional preferredJwsAlgorithm = Optional.empty(); private Optional grantType = Optional.empty(); @@ -173,6 +178,7 @@ public Builder from(final com.typesafe.config.Config configs) { useNonce = getOptional(configs, OIDC_USE_NONCE).map(Boolean::parseBoolean); customParamResource = getOptional(configs, OIDC_CUSTOM_PARAM_RESOURCE); readTimeout = getOptional(configs, OIDC_READ_TIMEOUT, DEFAULT_OIDC_READ_TIMEOUT); + connectTimeout = getOptional(configs, OIDC_CONNECT_TIMEOUT, DEFAULT_OIDC_CONNECT_TIMEOUT); extractJwtAccessTokenClaims = getOptional(configs, OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).map(Boolean::parseBoolean); preferredJwsAlgorithm = @@ -232,6 +238,9 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe if (jsonNode.has(READ_TIMEOUT)) { readTimeout = jsonNode.get(READ_TIMEOUT).asText(); } + if (jsonNode.has(CONNECT_TIMEOUT)) { + connectTimeout = jsonNode.get(CONNECT_TIMEOUT).asText(); + } if (jsonNode.has(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS)) { extractJwtAccessTokenClaims = Optional.of(jsonNode.get(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).asBoolean()); @@ -250,11 +259,11 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe } public OidcConfigs build() { - Objects.requireNonNull(_oidcEnabled, "oidcEnabled is required"); + Objects.requireNonNull(oidcEnabled, "oidcEnabled is required"); Objects.requireNonNull(clientId, "clientId is required"); Objects.requireNonNull(clientSecret, "clientSecret is required"); Objects.requireNonNull(discoveryUri, "discoveryUri is required"); - Objects.requireNonNull(_authBaseUrl, "authBaseUrl is required"); + Objects.requireNonNull(authBaseUrl, "authBaseUrl is required"); return new OidcConfigs(this); } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java index a8a3205e8299c..7fcaa5a9683cb 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcProvider.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcProvider.java @@ -2,15 +2,16 @@ import auth.sso.SsoProvider; import auth.sso.oidc.custom.CustomOidcClient; -import com.google.common.collect.ImmutableMap; +import com.nimbusds.jose.JWSAlgorithm; import java.util.HashMap; import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.pac4j.core.client.Client; import org.pac4j.core.http.callback.PathParameterCallbackUrlResolver; import org.pac4j.oidc.config.OidcConfiguration; -import org.pac4j.oidc.credentials.OidcCredentials; import org.pac4j.oidc.profile.OidcProfileDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Implementation of {@link SsoProvider} supporting the OIDC protocol. @@ -24,25 +25,25 @@ */ @Slf4j public class OidcProvider implements SsoProvider { - + private static final Logger logger = LoggerFactory.getLogger(OidcProvider.class); private static final String OIDC_CLIENT_NAME = "oidc"; - private final OidcConfigs _oidcConfigs; - private final Client _oidcClient; // Used primarily for redirecting to IdP. + private final OidcConfigs oidcConfigs; + private final Client oidcClient; // Used primarily for redirecting to IdP. public OidcProvider(final OidcConfigs configs) { - _oidcConfigs = configs; - _oidcClient = createPac4jClient(); + oidcConfigs = configs; + oidcClient = createPac4jClient(); } @Override - public Client client() { - return _oidcClient; + public Client client() { + return oidcClient; } @Override public OidcConfigs configs() { - return _oidcConfigs; + return oidcConfigs; } @Override @@ -50,50 +51,52 @@ public SsoProtocol protocol() { return SsoProtocol.OIDC; } - private Client createPac4jClient() { + private Client createPac4jClient() { final OidcConfiguration oidcConfiguration = new OidcConfiguration(); - oidcConfiguration.setClientId(_oidcConfigs.getClientId()); - oidcConfiguration.setSecret(_oidcConfigs.getClientSecret()); - oidcConfiguration.setDiscoveryURI(_oidcConfigs.getDiscoveryUri()); + oidcConfiguration.setClientId(oidcConfigs.getClientId()); + oidcConfiguration.setSecret(oidcConfigs.getClientSecret()); + oidcConfiguration.setDiscoveryURI(oidcConfigs.getDiscoveryUri()); oidcConfiguration.setClientAuthenticationMethodAsString( - _oidcConfigs.getClientAuthenticationMethod()); - oidcConfiguration.setScope(_oidcConfigs.getScope()); + oidcConfigs.getClientAuthenticationMethod()); + oidcConfiguration.setScope(oidcConfigs.getScope()); try { - oidcConfiguration.setReadTimeout(Integer.parseInt(_oidcConfigs.getReadTimeout())); + oidcConfiguration.setConnectTimeout(Integer.parseInt(oidcConfigs.getConnectTimeout())); + } catch (NumberFormatException e) { + log.warn("Invalid connect timeout configuration, defaulting to 1000ms"); + } + try { + oidcConfiguration.setReadTimeout(Integer.parseInt(oidcConfigs.getReadTimeout())); } catch (NumberFormatException e) { log.warn("Invalid read timeout configuration, defaulting to 5000ms"); } - _oidcConfigs.getResponseType().ifPresent(oidcConfiguration::setResponseType); - _oidcConfigs.getResponseMode().ifPresent(oidcConfiguration::setResponseMode); - _oidcConfigs.getUseNonce().ifPresent(oidcConfiguration::setUseNonce); + oidcConfigs.getResponseType().ifPresent(oidcConfiguration::setResponseType); + oidcConfigs.getResponseMode().ifPresent(oidcConfiguration::setResponseMode); + oidcConfigs.getUseNonce().ifPresent(oidcConfiguration::setUseNonce); Map customParamsMap = new HashMap<>(); - _oidcConfigs - .getCustomParamResource() - .ifPresent(value -> customParamsMap.put("resource", value)); - _oidcConfigs - .getGrantType() - .ifPresent(value -> customParamsMap.put("grant_type", value)); - _oidcConfigs - .getAcrValues() - .ifPresent(value -> customParamsMap.put("acr_values", value)); + oidcConfigs.getCustomParamResource().ifPresent(value -> customParamsMap.put("resource", value)); + oidcConfigs.getGrantType().ifPresent(value -> customParamsMap.put("grant_type", value)); + oidcConfigs.getAcrValues().ifPresent(value -> customParamsMap.put("acr_values", value)); if (!customParamsMap.isEmpty()) { oidcConfiguration.setCustomParams(customParamsMap); } - _oidcConfigs + oidcConfigs .getPreferredJwsAlgorithm() .ifPresent( preferred -> { log.info("Setting preferredJwsAlgorithm: " + preferred); - oidcConfiguration.setPreferredJwsAlgorithm(preferred); + oidcConfiguration.setPreferredJwsAlgorithm(JWSAlgorithm.parse(preferred)); }); + // Enable state parameter validation + oidcConfiguration.setWithState(true); + final CustomOidcClient oidcClient = new CustomOidcClient(oidcConfiguration); oidcClient.setName(OIDC_CLIENT_NAME); - oidcClient.setCallbackUrl( - _oidcConfigs.getAuthBaseUrl() + _oidcConfigs.getAuthBaseCallbackPath()); + oidcClient.setCallbackUrl(oidcConfigs.getAuthBaseUrl() + oidcConfigs.getAuthBaseCallbackPath()); oidcClient.setCallbackUrlResolver(new PathParameterCallbackUrlResolver()); oidcClient.addAuthorizationGenerator( - new OidcAuthorizationGenerator(new OidcProfileDefinition(), _oidcConfigs)); + new OidcAuthorizationGenerator(new OidcProfileDefinition(), oidcConfigs)); + return oidcClient; } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java index 9881b5e095b78..2843beee61610 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java @@ -4,7 +4,8 @@ import static play.mvc.Results.unauthorized; import java.util.Optional; -import org.pac4j.play.PlayWebContext; +import org.pac4j.core.context.CallContext; +import org.pac4j.core.context.WebContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import play.mvc.Result; @@ -13,14 +14,14 @@ public class OidcResponseErrorHandler { private OidcResponseErrorHandler() {} - private static final Logger _logger = LoggerFactory.getLogger("OidcResponseErrorHandler"); + private static final Logger logger = LoggerFactory.getLogger("OidcResponseErrorHandler"); private static final String ERROR_FIELD_NAME = "error"; private static final String ERROR_DESCRIPTION_FIELD_NAME = "error_description"; - public static Result handleError(final PlayWebContext context) { - - _logger.warn( + public static Result handleError(final CallContext ctx) { + WebContext context = ctx.webContext(); + logger.warn( "OIDC responded with an error: '{}'. Error description: '{}'", getError(context), getErrorDescription(context)); @@ -44,15 +45,15 @@ public static Result handleError(final PlayWebContext context) { getError(context).orElse(""), getErrorDescription(context).orElse(""))); } - public static boolean isError(final PlayWebContext context) { - return getError(context).isPresent() && !getError(context).get().isEmpty(); + public static boolean isError(final CallContext ctx) { + return getError(ctx.webContext()).isPresent() && !getError(ctx.webContext()).get().isEmpty(); } - public static Optional getError(final PlayWebContext context) { + public static Optional getError(final WebContext context) { return context.getRequestParameter(ERROR_FIELD_NAME); } - public static Optional getErrorDescription(final PlayWebContext context) { + public static Optional getErrorDescription(final WebContext context) { return context.getRequestParameter(ERROR_DESCRIPTION_FIELD_NAME); } } diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java index 01f8f16171d13..2288547cf6ed1 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java @@ -2,7 +2,6 @@ import com.nimbusds.oauth2.sdk.AuthorizationCode; import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant; -import com.nimbusds.oauth2.sdk.AuthorizationGrant; import com.nimbusds.oauth2.sdk.ParseException; import com.nimbusds.oauth2.sdk.TokenErrorResponse; import com.nimbusds.oauth2.sdk.TokenRequest; @@ -18,6 +17,7 @@ import com.nimbusds.oauth2.sdk.pkce.CodeVerifier; import com.nimbusds.openid.connect.sdk.OIDCTokenResponse; import com.nimbusds.openid.connect.sdk.OIDCTokenResponseParser; +import com.nimbusds.openid.connect.sdk.op.OIDCProviderMetadata; import com.nimbusds.openid.connect.sdk.token.OIDCTokens; import java.io.IOException; import java.net.URI; @@ -25,9 +25,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Optional; +import org.pac4j.core.context.CallContext; import org.pac4j.core.context.WebContext; -import org.pac4j.core.credentials.authenticator.Authenticator; +import org.pac4j.core.credentials.Credentials; import org.pac4j.core.exception.TechnicalException; import org.pac4j.core.util.CommonHelper; import org.pac4j.oidc.client.OidcClient; @@ -37,9 +39,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CustomOidcAuthenticator implements Authenticator { +public class CustomOidcAuthenticator extends OidcAuthenticator { - private static final Logger logger = LoggerFactory.getLogger(OidcAuthenticator.class); + private static final Logger logger = LoggerFactory.getLogger(CustomOidcAuthenticator.class); private static final Collection SUPPORTED_METHODS = Arrays.asList( @@ -47,21 +49,24 @@ public class CustomOidcAuthenticator implements Authenticator { ClientAuthenticationMethod.CLIENT_SECRET_BASIC, ClientAuthenticationMethod.NONE); - protected OidcConfiguration configuration; - - protected OidcClient client; - private final ClientAuthentication clientAuthentication; - public CustomOidcAuthenticator(final OidcClient client) { - CommonHelper.assertNotNull("configuration", client.getConfiguration()); - CommonHelper.assertNotNull("client", client); - this.configuration = client.getConfiguration(); - this.client = client; + public CustomOidcAuthenticator(final OidcClient client) { + super(client.getConfiguration(), client); // check authentication methods - final List metadataMethods = - configuration.findProviderMetadata().getTokenEndpointAuthMethods(); + OIDCProviderMetadata providerMetadata; + try { + providerMetadata = loadWithRetry(); + } catch (TechnicalException e) { + logger.error( + "Could not resolve identity provider's remote configuration from DiscoveryURI: {}", + configuration.getDiscoveryURI()); + throw e; + } + + List metadataMethods = + providerMetadata.getTokenEndpointAuthMethods(); final ClientAuthenticationMethod preferredMethod = getPreferredAuthenticationMethod(configuration); @@ -146,8 +151,11 @@ private static ClientAuthenticationMethod firstSupportedMethod( } @Override - public void validate(final OidcCredentials credentials, final WebContext context) { - final AuthorizationCode code = credentials.getCode(); + public Optional validate(CallContext ctx, Credentials cred) { + OidcCredentials credentials = (OidcCredentials) cred; + WebContext context = ctx.webContext(); + + final AuthorizationCode code = credentials.toAuthorizationCode(); // if we have a code if (code != null) { try { @@ -156,7 +164,7 @@ public void validate(final OidcCredentials credentials, final WebContext context (CodeVerifier) configuration .getValueRetriever() - .retrieve(client.getCodeVerifierSessionAttributeName(), client, context) + .retrieve(ctx, client.getCodeVerifierSessionAttributeName(), client) .orElse(null); // Token request final TokenRequest request = @@ -182,27 +190,49 @@ public void validate(final OidcCredentials credentials, final WebContext context // save tokens in credentials final OIDCTokens oidcTokens = tokenSuccessResponse.getOIDCTokens(); - credentials.setAccessToken(oidcTokens.getAccessToken()); - credentials.setRefreshToken(oidcTokens.getRefreshToken()); - credentials.setIdToken(oidcTokens.getIDToken()); + credentials.setAccessTokenObject(oidcTokens.getAccessToken()); + + // Only set refresh token if it exists + if (oidcTokens.getRefreshToken() != null) { + credentials.setRefreshTokenObject(oidcTokens.getRefreshToken()); + } + + if (oidcTokens.getIDToken() != null) { + credentials.setIdToken(oidcTokens.getIDToken().getParsedString()); + } } catch (final URISyntaxException | IOException | ParseException e) { throw new TechnicalException(e); } } + + return Optional.ofNullable(cred); } - private TokenRequest createTokenRequest(final AuthorizationGrant grant) { - if (clientAuthentication != null) { - return new TokenRequest( - configuration.findProviderMetadata().getTokenEndpointURI(), - this.clientAuthentication, - grant); - } else { - return new TokenRequest( - configuration.findProviderMetadata().getTokenEndpointURI(), - new ClientID(configuration.getClientId()), - grant); + // Simple retry with exponential backoff + public OIDCProviderMetadata loadWithRetry() { + int maxAttempts = 3; + long initialDelay = 1000; // 1 second + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + OIDCProviderMetadata providerMetadata = configuration.getOpMetadataResolver().load(); + return Objects.requireNonNull(providerMetadata); + } catch (RuntimeException e) { + if (attempt == maxAttempts) { + throw e; // Rethrow on final attempt + } + try { + // Exponential backoff + Thread.sleep(initialDelay * (long) Math.pow(2, attempt - 1)); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Retry interrupted", ie); + } + logger.warn("Retry attempt {} of {} failed", attempt, maxAttempts, e); + } } + throw new RuntimeException( + "Failed to load provider metadata after " + maxAttempts + " attempts"); } } diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java index 3a0a247cb761e..28c927c2aa541 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcClient.java @@ -3,25 +3,38 @@ import org.pac4j.core.util.CommonHelper; import org.pac4j.oidc.client.OidcClient; import org.pac4j.oidc.config.OidcConfiguration; -import org.pac4j.oidc.credentials.extractor.OidcExtractor; +import org.pac4j.oidc.credentials.extractor.OidcCredentialsExtractor; import org.pac4j.oidc.logout.OidcLogoutActionBuilder; +import org.pac4j.oidc.logout.processor.OidcLogoutProcessor; import org.pac4j.oidc.profile.creator.OidcProfileCreator; -import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder; -public class CustomOidcClient extends OidcClient { +public class CustomOidcClient extends OidcClient { - public CustomOidcClient(final OidcConfiguration configuration) { - setConfiguration(configuration); + public CustomOidcClient(OidcConfiguration configuration) { + super(configuration); } @Override - protected void clientInit() { + protected void internalInit(final boolean forceReinit) { + // Validate configuration CommonHelper.assertNotNull("configuration", getConfiguration()); - getConfiguration().init(); - defaultRedirectionActionBuilder(new CustomOidcRedirectionActionBuilder(getConfiguration(), this)); - defaultCredentialsExtractor(new OidcExtractor(getConfiguration(), this)); - defaultAuthenticator(new CustomOidcAuthenticator(this)); - defaultProfileCreator(new OidcProfileCreator<>(getConfiguration(), this)); - defaultLogoutActionBuilder(new OidcLogoutActionBuilder(getConfiguration())); + + // Initialize configuration + getConfiguration().init(forceReinit); + + // Initialize client components + setRedirectionActionBuilderIfUndefined( + new CustomOidcRedirectionActionBuilder(getConfiguration(), this)); + setCredentialsExtractorIfUndefined(new OidcCredentialsExtractor(getConfiguration(), this)); + + // Initialize default authenticator if not set + if (getAuthenticator() == null || forceReinit) { + setAuthenticatorIfUndefined(new CustomOidcAuthenticator(this)); + } + + setProfileCreatorIfUndefined(new OidcProfileCreator(getConfiguration(), this)); + setLogoutProcessorIfUndefined( + new OidcLogoutProcessor(getConfiguration(), findSessionLogoutHandler())); + setLogoutActionBuilderIfUndefined(new OidcLogoutActionBuilder(getConfiguration())); } } diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java index bdeeacc895af3..ea5315972344d 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcRedirectionActionBuilder.java @@ -2,29 +2,35 @@ import java.util.Map; import java.util.Optional; +import org.pac4j.core.context.CallContext; import org.pac4j.core.context.WebContext; import org.pac4j.core.exception.http.RedirectionAction; -import org.pac4j.core.exception.http.RedirectionActionHelper; +import org.pac4j.core.util.HttpActionHelper; import org.pac4j.oidc.client.OidcClient; import org.pac4j.oidc.config.OidcConfiguration; import org.pac4j.oidc.redirect.OidcRedirectionActionBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class CustomOidcRedirectionActionBuilder extends OidcRedirectionActionBuilder { private static final Logger logger = LoggerFactory.getLogger(OidcRedirectionActionBuilder.class); + + private final OidcConfiguration configuration; + public CustomOidcRedirectionActionBuilder(OidcConfiguration configuration, OidcClient client) { - super(configuration, client); + super(client); + this.configuration = configuration; } @Override - public Optional getRedirectionAction(WebContext context) { - Map params = this.buildParams(); + public Optional getRedirectionAction(CallContext ctx) { + WebContext context = ctx.webContext(); + + Map params = this.buildParams(context); String computedCallbackUrl = this.client.computeFinalCallbackUrl(context); params.put("redirect_uri", computedCallbackUrl); - this.addStateAndNonceParameters(context, params); + this.addStateAndNonceParameters(ctx, params); if (this.configuration.getMaxAge() != null) { params.put("max_age", this.configuration.getMaxAge().toString()); } @@ -40,7 +46,6 @@ public Optional getRedirectionAction(WebContext context) { } logger.debug("Authentication request url: {}", location); - return Optional.of(RedirectionActionHelper.buildRedirectUrlAction(context, location)); + return Optional.of(HttpActionHelper.buildRedirectUrlAction(context, location)); } - } diff --git a/datahub-frontend/app/client/KafkaTrackingProducer.java b/datahub-frontend/app/client/KafkaTrackingProducer.java index 058e75100c24a..a29fe3bb7aef7 100644 --- a/datahub-frontend/app/client/KafkaTrackingProducer.java +++ b/datahub-frontend/app/client/KafkaTrackingProducer.java @@ -27,7 +27,8 @@ @Singleton public class KafkaTrackingProducer { - private final Logger _logger = LoggerFactory.getLogger(KafkaTrackingProducer.class.getName()); + private static final Logger logger = + LoggerFactory.getLogger(KafkaTrackingProducer.class.getName()); private static final List KAFKA_SSL_PROTOCOLS = Collections.unmodifiableList( Arrays.asList( @@ -35,38 +36,38 @@ public class KafkaTrackingProducer { SecurityProtocol.SASL_SSL.name(), SecurityProtocol.SASL_PLAINTEXT.name())); - private final Boolean _isEnabled; - private final KafkaProducer _producer; + private final Boolean isEnabled; + private final KafkaProducer producer; @Inject public KafkaTrackingProducer( @Nonnull Config config, ApplicationLifecycle lifecycle, final ConfigurationProvider configurationProvider) { - _isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled"); + isEnabled = !config.hasPath("analytics.enabled") || config.getBoolean("analytics.enabled"); - if (_isEnabled) { - _logger.debug("Analytics tracking is enabled"); - _producer = createKafkaProducer(config, configurationProvider.getKafka()); + if (isEnabled) { + logger.debug("Analytics tracking is enabled"); + producer = createKafkaProducer(config, configurationProvider.getKafka()); lifecycle.addStopHook( () -> { - _producer.flush(); - _producer.close(); + producer.flush(); + producer.close(); return CompletableFuture.completedFuture(null); }); } else { - _logger.debug("Analytics tracking is disabled"); - _producer = null; + logger.debug("Analytics tracking is disabled"); + producer = null; } } public Boolean isEnabled() { - return _isEnabled; + return isEnabled; } public void send(ProducerRecord record) { - _producer.send(record); + producer.send(record); } private static KafkaProducer createKafkaProducer( diff --git a/datahub-frontend/app/config/ConfigurationProvider.java b/datahub-frontend/app/config/ConfigurationProvider.java index d447b28cdcc46..97e916769a6c4 100644 --- a/datahub-frontend/app/config/ConfigurationProvider.java +++ b/datahub-frontend/app/config/ConfigurationProvider.java @@ -6,12 +6,9 @@ import com.linkedin.metadata.config.kafka.KafkaConfiguration; import com.linkedin.metadata.spring.YamlPropertySourceFactory; import lombok.Data; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import org.springframework.stereotype.Component; /** * Minimal sharing between metadata-service and frontend Does not use the factories module to avoid diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java index 017847367de05..1c3786f4af526 100644 --- a/datahub-frontend/app/controllers/Application.java +++ b/datahub-frontend/app/controllers/Application.java @@ -36,8 +36,8 @@ import play.libs.ws.StandaloneWSClient; import play.libs.ws.ahc.StandaloneAhcWSClient; import play.mvc.Controller; -import play.mvc.Http.Cookie; import play.mvc.Http; +import play.mvc.Http.Cookie; import play.mvc.ResponseHeader; import play.mvc.Result; import play.mvc.Security; @@ -48,16 +48,16 @@ import utils.ConfigUtil; public class Application extends Controller { - private final Logger _logger = LoggerFactory.getLogger(Application.class.getName()); - private final Config _config; - private final StandaloneWSClient _ws; - private final Environment _environment; + private static final Logger logger = LoggerFactory.getLogger(Application.class.getName()); + private final Config config; + private final StandaloneWSClient ws; + private final Environment environment; @Inject public Application(Environment environment, @Nonnull Config config) { - _config = config; - _ws = createWsClient(); - _environment = environment; + this.config = config; + ws = createWsClient(); + this.environment = environment; } /** @@ -69,10 +69,10 @@ public Application(Environment environment, @Nonnull Config config) { @Nonnull private Result serveAsset(@Nullable String path) { try { - InputStream indexHtml = _environment.resourceAsStream("public/index.html"); + InputStream indexHtml = environment.resourceAsStream("public/index.html"); return ok(indexHtml).withHeader("Cache-Control", "no-cache").as("text/html"); } catch (Exception e) { - _logger.warn("Cannot load public/index.html resource. Static assets or assets jar missing?"); + logger.warn("Cannot load public/index.html resource. Static assets or assets jar missing?"); return notFound().withHeader("Cache-Control", "no-cache").as("text/html"); } } @@ -106,17 +106,17 @@ public CompletableFuture proxy(String path, Http.Request request) final String metadataServiceHost = ConfigUtil.getString( - _config, + config, ConfigUtil.METADATA_SERVICE_HOST_CONFIG_PATH, ConfigUtil.DEFAULT_METADATA_SERVICE_HOST); final int metadataServicePort = ConfigUtil.getInt( - _config, + config, ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH, ConfigUtil.DEFAULT_METADATA_SERVICE_PORT); final boolean metadataServiceUseSsl = ConfigUtil.getBoolean( - _config, + config, ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH, ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL); @@ -139,7 +139,7 @@ public CompletableFuture proxy(String path, Http.Request request) // Get the current time to measure the duration of the request Instant start = Instant.now(); - return _ws.url( + return ws.url( String.format( "%s://%s:%s%s", protocol, metadataServiceHost, metadataServicePort, resolvedUri)) .setMethod(request.method()) @@ -167,9 +167,10 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req .execute() .thenApply( apiResponse -> { - // Log the query if it takes longer than the configured threshold and verbose logging is enabled - boolean verboseGraphQLLogging = _config.getBoolean("graphql.verbose.logging"); - int verboseGraphQLLongQueryMillis = _config.getInt("graphql.verbose.slowQueryMillis"); + // Log the query if it takes longer than the configured threshold and verbose logging + // is enabled + boolean verboseGraphQLLogging = config.getBoolean("graphql.verbose.logging"); + int verboseGraphQLLongQueryMillis = config.getInt("graphql.verbose.slowQueryMillis"); Instant finish = Instant.now(); long timeElapsed = Duration.between(start, finish).toMillis(); if (verboseGraphQLLogging && timeElapsed >= verboseGraphQLLongQueryMillis) { @@ -206,32 +207,32 @@ AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER, getDataHubActorHeader(req public Result appConfig() { final ObjectNode config = Json.newObject(); config.put("application", "datahub-frontend"); - config.put("appVersion", _config.getString("app.version")); - config.put("isInternal", _config.getBoolean("linkedin.internal")); - config.put("shouldShowDatasetLineage", _config.getBoolean("linkedin.show.dataset.lineage")); + config.put("appVersion", this.config.getString("app.version")); + config.put("isInternal", this.config.getBoolean("linkedin.internal")); + config.put("shouldShowDatasetLineage", this.config.getBoolean("linkedin.show.dataset.lineage")); config.put( "suggestionConfidenceThreshold", - Integer.valueOf(_config.getString("linkedin.suggestion.confidence.threshold"))); + Integer.valueOf(this.config.getString("linkedin.suggestion.confidence.threshold"))); config.set("wikiLinks", wikiLinks()); config.set("tracking", trackingInfo()); // In a staging environment, we can trigger this flag to be true so that the UI can handle based // on // such config and alert users that their changes will not affect production data - config.put("isStagingBanner", _config.getBoolean("ui.show.staging.banner")); - config.put("isLiveDataWarning", _config.getBoolean("ui.show.live.data.banner")); - config.put("showChangeManagement", _config.getBoolean("ui.show.CM.banner")); + config.put("isStagingBanner", this.config.getBoolean("ui.show.staging.banner")); + config.put("isLiveDataWarning", this.config.getBoolean("ui.show.live.data.banner")); + config.put("showChangeManagement", this.config.getBoolean("ui.show.CM.banner")); // Flag to enable people entity elements - config.put("showPeople", _config.getBoolean("ui.show.people")); - config.put("changeManagementLink", _config.getString("ui.show.CM.link")); + config.put("showPeople", this.config.getBoolean("ui.show.people")); + config.put("changeManagementLink", this.config.getString("ui.show.CM.link")); // Flag set in order to warn users that search is experiencing issues - config.put("isStaleSearch", _config.getBoolean("ui.show.stale.search")); - config.put("showAdvancedSearch", _config.getBoolean("ui.show.advanced.search")); + config.put("isStaleSearch", this.config.getBoolean("ui.show.stale.search")); + config.put("showAdvancedSearch", this.config.getBoolean("ui.show.advanced.search")); // Flag to use the new api for browsing datasets - config.put("useNewBrowseDataset", _config.getBoolean("ui.new.browse.dataset")); + config.put("useNewBrowseDataset", this.config.getBoolean("ui.new.browse.dataset")); // show lineage graph in relationships tabs - config.put("showLineageGraph", _config.getBoolean("ui.show.lineage.graph")); + config.put("showLineageGraph", this.config.getBoolean("ui.show.lineage.graph")); // show institutional memory for available entities - config.put("showInstitutionalMemory", _config.getBoolean("ui.show.institutional.memory")); + config.put("showInstitutionalMemory", this.config.getBoolean("ui.show.institutional.memory")); // Insert properties for user profile operations config.set("userEntityProps", userEntityProps()); @@ -250,8 +251,8 @@ public Result appConfig() { @Nonnull private ObjectNode userEntityProps() { final ObjectNode props = Json.newObject(); - props.put("aviUrlPrimary", _config.getString("linkedin.links.avi.urlPrimary")); - props.put("aviUrlFallback", _config.getString("linkedin.links.avi.urlFallback")); + props.put("aviUrlPrimary", config.getString("linkedin.links.avi.urlPrimary")); + props.put("aviUrlFallback", config.getString("linkedin.links.avi.urlFallback")); return props; } @@ -261,19 +262,19 @@ private ObjectNode userEntityProps() { @Nonnull private ObjectNode wikiLinks() { final ObjectNode wikiLinks = Json.newObject(); - wikiLinks.put("appHelp", _config.getString("links.wiki.appHelp")); - wikiLinks.put("gdprPii", _config.getString("links.wiki.gdprPii")); - wikiLinks.put("tmsSchema", _config.getString("links.wiki.tmsSchema")); - wikiLinks.put("gdprTaxonomy", _config.getString("links.wiki.gdprTaxonomy")); - wikiLinks.put("staleSearchIndex", _config.getString("links.wiki.staleSearchIndex")); - wikiLinks.put("dht", _config.getString("links.wiki.dht")); - wikiLinks.put("purgePolicies", _config.getString("links.wiki.purgePolicies")); - wikiLinks.put("jitAcl", _config.getString("links.wiki.jitAcl")); - wikiLinks.put("metadataCustomRegex", _config.getString("links.wiki.metadataCustomRegex")); - wikiLinks.put("exportPolicy", _config.getString("links.wiki.exportPolicy")); - wikiLinks.put("metadataHealth", _config.getString("links.wiki.metadataHealth")); - wikiLinks.put("purgeKey", _config.getString("links.wiki.purgeKey")); - wikiLinks.put("datasetDecommission", _config.getString("links.wiki.datasetDecommission")); + wikiLinks.put("appHelp", config.getString("links.wiki.appHelp")); + wikiLinks.put("gdprPii", config.getString("links.wiki.gdprPii")); + wikiLinks.put("tmsSchema", config.getString("links.wiki.tmsSchema")); + wikiLinks.put("gdprTaxonomy", config.getString("links.wiki.gdprTaxonomy")); + wikiLinks.put("staleSearchIndex", config.getString("links.wiki.staleSearchIndex")); + wikiLinks.put("dht", config.getString("links.wiki.dht")); + wikiLinks.put("purgePolicies", config.getString("links.wiki.purgePolicies")); + wikiLinks.put("jitAcl", config.getString("links.wiki.jitAcl")); + wikiLinks.put("metadataCustomRegex", config.getString("links.wiki.metadataCustomRegex")); + wikiLinks.put("exportPolicy", config.getString("links.wiki.exportPolicy")); + wikiLinks.put("metadataHealth", config.getString("links.wiki.metadataHealth")); + wikiLinks.put("purgeKey", config.getString("links.wiki.purgeKey")); + wikiLinks.put("datasetDecommission", config.getString("links.wiki.datasetDecommission")); return wikiLinks; } @@ -283,8 +284,8 @@ private ObjectNode wikiLinks() { @Nonnull private ObjectNode trackingInfo() { final ObjectNode piwik = Json.newObject(); - piwik.put("piwikSiteId", Integer.valueOf(_config.getString("tracking.piwik.siteid"))); - piwik.put("piwikUrl", _config.getString("tracking.piwik.url")); + piwik.put("piwikSiteId", Integer.valueOf(config.getString("tracking.piwik.siteid"))); + piwik.put("piwikUrl", config.getString("tracking.piwik.url")); final ObjectNode trackers = Json.newObject(); trackers.set("piwik", piwik); @@ -376,9 +377,10 @@ private String mapPath(@Nonnull final String path) { return path; } - /** - * Called if verbose logging is enabled and request takes longer that the slow query milliseconds defined in the config + * Called if verbose logging is enabled and request takes longer that the slow query milliseconds + * defined in the config + * * @param request GraphQL request that was made * @param resolvedUri URI that was requested * @param duration How long the query took to complete @@ -393,16 +395,16 @@ private void logSlowQuery(Http.Request request, String resolvedUri, float durati JsonNode jsonNode = request.body().asJson(); ((ObjectNode) jsonNode).remove("query"); jsonBody.append(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonNode)); - } - catch (Exception e) { - _logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri); + } catch (Exception e) { + logger.info("GraphQL Request Received: {}, Unable to parse JSON body", resolvedUri); } String jsonBodyStr = jsonBody.toString(); - _logger.info("Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms", - resolvedUri, - request.queryString(), - actorValue, - jsonBodyStr, - duration); + logger.info( + "Slow GraphQL Request Received: {}, Request query string: {}, Request actor: {}, Request JSON: {}, Request completed in {} ms", + resolvedUri, + request.queryString(), + actorValue, + jsonBodyStr, + duration); } } diff --git a/datahub-frontend/app/controllers/AuthenticationController.java b/datahub-frontend/app/controllers/AuthenticationController.java index 87c4b5ba06793..3c8ace9aee29f 100644 --- a/datahub-frontend/app/controllers/AuthenticationController.java +++ b/datahub-frontend/app/controllers/AuthenticationController.java @@ -26,12 +26,14 @@ import org.apache.commons.httpclient.InvalidRedirectLocationException; import org.apache.commons.lang3.StringUtils; import org.pac4j.core.client.Client; +import org.pac4j.core.context.CallContext; import org.pac4j.core.context.Cookie; +import org.pac4j.core.context.WebContext; import org.pac4j.core.exception.http.FoundAction; import org.pac4j.core.exception.http.RedirectionAction; import org.pac4j.play.PlayWebContext; import org.pac4j.play.http.PlayHttpActionAdapter; -import org.pac4j.play.store.PlaySessionStore; +import org.pac4j.play.store.PlayCookieSessionStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import play.data.validation.Constraints; @@ -51,26 +53,27 @@ public class AuthenticationController extends Controller { private static final String SSO_NO_REDIRECT_MESSAGE = "SSO is configured, however missing redirect from idp"; - private final Logger _logger = LoggerFactory.getLogger(AuthenticationController.class.getName()); - private final CookieConfigs _cookieConfigs; - private final JAASConfigs _jaasConfigs; - private final NativeAuthenticationConfigs _nativeAuthenticationConfigs; - private final boolean _verbose; + private static final Logger logger = + LoggerFactory.getLogger(AuthenticationController.class.getName()); + private final CookieConfigs cookieConfigs; + private final JAASConfigs jaasConfigs; + private final NativeAuthenticationConfigs nativeAuthenticationConfigs; + private final boolean verbose; - @Inject private org.pac4j.core.config.Config _ssoConfig; + @Inject private org.pac4j.core.config.Config ssoConfig; - @Inject private PlaySessionStore _playSessionStore; + @Inject private PlayCookieSessionStore playCookieSessionStore; - @Inject private SsoManager _ssoManager; + @Inject private SsoManager ssoManager; - @Inject AuthServiceClient _authClient; + @Inject AuthServiceClient authClient; @Inject public AuthenticationController(@Nonnull Config configs) { - _cookieConfigs = new CookieConfigs(configs); - _jaasConfigs = new JAASConfigs(configs); - _nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs); - _verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING); + cookieConfigs = new CookieConfigs(configs); + jaasConfigs = new JAASConfigs(configs); + nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs); + verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING); } /** @@ -92,11 +95,14 @@ public Result authenticate(Http.Request request) { try { URI redirectUri = new URI(redirectPath); if (redirectUri.getScheme() != null || redirectUri.getAuthority() != null) { - throw new InvalidRedirectLocationException("Redirect location must be relative to the base url, cannot " - + "redirect to other domains: " + redirectPath, redirectPath); + throw new InvalidRedirectLocationException( + "Redirect location must be relative to the base url, cannot " + + "redirect to other domains: " + + redirectPath, + redirectPath); } } catch (URISyntaxException | InvalidRedirectLocationException e) { - _logger.warn(e.getMessage()); + logger.warn(e.getMessage()); redirectPath = "/"; } @@ -105,7 +111,7 @@ public Result authenticate(Http.Request request) { } // 1. If SSO is enabled, redirect to IdP if not authenticated. - if (_ssoManager.isSsoEnabled()) { + if (ssoManager.isSsoEnabled()) { return redirectToIdentityProvider(request, redirectPath) .orElse( Results.redirect( @@ -114,8 +120,8 @@ public Result authenticate(Http.Request request) { } // 2. If either JAAS auth or Native auth is enabled, fallback to it - if (_jaasConfigs.isJAASEnabled() - || _nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) { + if (jaasConfigs.isJAASEnabled() + || nativeAuthenticationConfigs.isNativeAuthenticationEnabled()) { return Results.redirect( LOGIN_ROUTE + String.format("?%s=%s", AUTH_REDIRECT_URI_PARAM, encodeRedirectUri(redirectPath))); @@ -123,21 +129,21 @@ public Result authenticate(Http.Request request) { // 3. If no auth enabled, fallback to using default user account & redirect. // Generate GMS session token, TODO: - final String accessToken = _authClient.generateSessionTokenForUser(DEFAULT_ACTOR_URN.getId()); + final String accessToken = authClient.generateSessionTokenForUser(DEFAULT_ACTOR_URN.getId()); return Results.redirect(redirectPath) .withSession(createSessionMap(DEFAULT_ACTOR_URN.toString(), accessToken)) .withCookies( createActorCookie( DEFAULT_ACTOR_URN.toString(), - _cookieConfigs.getTtlInHours(), - _cookieConfigs.getAuthCookieSameSite(), - _cookieConfigs.getAuthCookieSecure())); + cookieConfigs.getTtlInHours(), + cookieConfigs.getAuthCookieSameSite(), + cookieConfigs.getAuthCookieSecure())); } /** Redirect to the identity provider for authentication. */ @Nonnull public Result sso(Http.Request request) { - if (_ssoManager.isSsoEnabled()) { + if (ssoManager.isSsoEnabled()) { return redirectToIdentityProvider(request, "/") .orElse( Results.redirect( @@ -156,11 +162,11 @@ public Result sso(Http.Request request) { */ @Nonnull public Result logIn(Http.Request request) { - boolean jaasEnabled = _jaasConfigs.isJAASEnabled(); - _logger.debug(String.format("Jaas authentication enabled: %b", jaasEnabled)); + boolean jaasEnabled = jaasConfigs.isJAASEnabled(); + logger.debug(String.format("Jaas authentication enabled: %b", jaasEnabled)); boolean nativeAuthenticationEnabled = - _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); - _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); boolean noAuthEnabled = !jaasEnabled && !nativeAuthenticationEnabled; if (noAuthEnabled) { String message = "Neither JAAS nor native authentication is enabled on the server."; @@ -182,13 +188,13 @@ public Result logIn(Http.Request request) { boolean loginSucceeded = tryLogin(username, password); if (!loginSucceeded) { - _logger.info("Login failed for user: {}", username); + logger.info("Login failed for user: {}", username); return Results.badRequest(invalidCredsJson); } final Urn actorUrn = new CorpuserUrn(username); - _logger.info("Login successful for user: {}, urn: {}", username, actorUrn); - final String accessToken = _authClient.generateSessionTokenForUser(actorUrn.getId()); + logger.info("Login successful for user: {}, urn: {}", username, actorUrn); + final String accessToken = authClient.generateSessionTokenForUser(actorUrn.getId()); return createSession(actorUrn.toString(), accessToken); } @@ -199,8 +205,8 @@ public Result logIn(Http.Request request) { @Nonnull public Result signUp(Http.Request request) { boolean nativeAuthenticationEnabled = - _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); - _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); if (!nativeAuthenticationEnabled) { String message = "Native authentication is not enabled on the server."; final ObjectNode error = Json.newObject(); @@ -224,7 +230,7 @@ public Result signUp(Http.Request request) { JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); return Results.badRequest(invalidCredsJson); } - if (_nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) { + if (nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) { Constraints.EmailValidator emailValidator = new Constraints.EmailValidator(); if (!emailValidator.isValid(email)) { JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty."); @@ -250,9 +256,9 @@ public Result signUp(Http.Request request) { final Urn userUrn = new CorpuserUrn(email); final String userUrnString = userUrn.toString(); - _authClient.signUp(userUrnString, fullName, email, title, password, inviteToken); - _logger.info("Signed up user {} using invite tokens", userUrnString); - final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId()); + authClient.signUp(userUrnString, fullName, email, title, password, inviteToken); + logger.info("Signed up user {} using invite tokens", userUrnString); + final String accessToken = authClient.generateSessionTokenForUser(userUrn.getId()); return createSession(userUrnString, accessToken); } @@ -260,8 +266,8 @@ public Result signUp(Http.Request request) { @Nonnull public Result resetNativeUserCredentials(Http.Request request) { boolean nativeAuthenticationEnabled = - _nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); - _logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); + nativeAuthenticationConfigs.isNativeAuthenticationEnabled(); + logger.debug(String.format("Native authentication enabled: %b", nativeAuthenticationEnabled)); if (!nativeAuthenticationEnabled) { String message = "Native authentication is not enabled on the server."; final ObjectNode error = Json.newObject(); @@ -291,26 +297,27 @@ public Result resetNativeUserCredentials(Http.Request request) { final Urn userUrn = new CorpuserUrn(email); final String userUrnString = userUrn.toString(); - _authClient.resetNativeUserCredentials(userUrnString, password, resetToken); - final String accessToken = _authClient.generateSessionTokenForUser(userUrn.getId()); + authClient.resetNativeUserCredentials(userUrnString, password, resetToken); + final String accessToken = authClient.generateSessionTokenForUser(userUrn.getId()); return createSession(userUrnString, accessToken); } private Optional redirectToIdentityProvider( Http.RequestHeader request, String redirectPath) { - final PlayWebContext playWebContext = new PlayWebContext(request, _playSessionStore); - final Client client = _ssoManager.getSsoProvider().client(); - configurePac4jSessionStore(playWebContext, client, redirectPath); + CallContext ctx = buildCallContext(request); + + final Client client = ssoManager.getSsoProvider().client(); + configurePac4jSessionStore(ctx, client, redirectPath); try { - final Optional action = client.getRedirectionAction(playWebContext); - return action.map(act -> new PlayHttpActionAdapter().adapt(act, playWebContext)); + final Optional action = client.getRedirectionAction(ctx); + return action.map(act -> new PlayHttpActionAdapter().adapt(act, ctx.webContext())); } catch (Exception e) { - if (_verbose) { - _logger.error( + if (verbose) { + logger.error( "Caught exception while attempting to redirect to SSO identity provider! It's likely that SSO integration is mis-configured", e); } else { - _logger.error( + logger.error( "Caught exception while attempting to redirect to SSO identity provider! It's likely that SSO integration is mis-configured"); } return Optional.of( @@ -324,22 +331,33 @@ private Optional redirectToIdentityProvider( } } - private void configurePac4jSessionStore( - PlayWebContext context, Client client, String redirectPath) { + private CallContext buildCallContext(Http.RequestHeader request) { + // First create PlayWebContext from the request + PlayWebContext webContext = new PlayWebContext(request); + + // Then create CallContext using the web context and session store + return new CallContext(webContext, playCookieSessionStore); + } + + private void configurePac4jSessionStore(CallContext ctx, Client client, String redirectPath) { + WebContext context = ctx.webContext(); + // Set the originally requested path for post-auth redirection. We split off into a separate // cookie from the session // to reduce size of the session cookie FoundAction foundAction = new FoundAction(redirectPath); - byte[] javaSerBytes = JAVA_SER_HELPER.serializeToBytes(foundAction); + byte[] javaSerBytes = + ((PlayCookieSessionStore) ctx.sessionStore()).getSerializer().serializeToBytes(foundAction); String serialized = Base64.getEncoder().encodeToString(compressBytes(javaSerBytes)); context.addResponseCookie(new Cookie(REDIRECT_URL_COOKIE_NAME, serialized)); // This is to prevent previous login attempts from being cached. // We replicate the logic here, which is buried in the Pac4j client. - if (_playSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX) - != null) { - _logger.debug( + Optional attempt = + playCookieSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX); + if (attempt.isPresent() && !"".equals(attempt.get())) { + logger.debug( "Found previous login attempt. Removing it manually to prevent unexpected errors."); - _playSessionStore.set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, ""); + playCookieSessionStore.set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, ""); } } @@ -351,27 +369,27 @@ private boolean tryLogin(String username, String password) { boolean loginSucceeded = false; // First try jaas login, if enabled - if (_jaasConfigs.isJAASEnabled()) { + if (jaasConfigs.isJAASEnabled()) { try { - _logger.debug("Attempting JAAS authentication for user: {}", username); + logger.debug("Attempting JAAS authentication for user: {}", username); AuthenticationManager.authenticateJaasUser(username, password); - _logger.debug("JAAS authentication successful. Login succeeded"); + logger.debug("JAAS authentication successful. Login succeeded"); loginSucceeded = true; } catch (Exception e) { - if (_verbose) { - _logger.debug("JAAS authentication error. Login failed", e); + if (verbose) { + logger.debug("JAAS authentication error. Login failed", e); } else { - _logger.debug("JAAS authentication error. Login failed"); + logger.debug("JAAS authentication error. Login failed"); } } } // If jaas login fails or is disabled, try native auth login - if (_nativeAuthenticationConfigs.isNativeAuthenticationEnabled() && !loginSucceeded) { + if (nativeAuthenticationConfigs.isNativeAuthenticationEnabled() && !loginSucceeded) { final Urn userUrn = new CorpuserUrn(username); final String userUrnString = userUrn.toString(); loginSucceeded = - loginSucceeded || _authClient.verifyNativeUserCredentials(userUrnString, password); + loginSucceeded || authClient.verifyNativeUserCredentials(userUrnString, password); } return loginSucceeded; @@ -383,8 +401,8 @@ private Result createSession(String userUrnString, String accessToken) { .withCookies( createActorCookie( userUrnString, - _cookieConfigs.getTtlInHours(), - _cookieConfigs.getAuthCookieSameSite(), - _cookieConfigs.getAuthCookieSecure())); + cookieConfigs.getTtlInHours(), + cookieConfigs.getAuthCookieSameSite(), + cookieConfigs.getAuthCookieSecure())); } } diff --git a/datahub-frontend/app/controllers/CentralLogoutController.java b/datahub-frontend/app/controllers/CentralLogoutController.java index eea1c662ebf89..d2e3027236adc 100644 --- a/datahub-frontend/app/controllers/CentralLogoutController.java +++ b/datahub-frontend/app/controllers/CentralLogoutController.java @@ -1,6 +1,6 @@ package controllers; -import com.typesafe.config.Config; +import auth.sso.SsoManager; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import javax.inject.Inject; @@ -15,13 +15,10 @@ public class CentralLogoutController extends LogoutController { private static final String AUTH_URL_CONFIG_PATH = "/login"; private static final String DEFAULT_BASE_URL_PATH = "/"; - private static Boolean _isOidcEnabled = false; + @Inject private SsoManager ssoManager; - @Inject - public CentralLogoutController(Config config) { - _isOidcEnabled = config.hasPath("auth.oidc.enabled") && config.getBoolean("auth.oidc.enabled"); - - setDefaultUrl(DEFAULT_BASE_URL_PATH); + public CentralLogoutController() { + setDefaultUrl(AUTH_URL_CONFIG_PATH); setLogoutUrlPattern(DEFAULT_BASE_URL_PATH + ".*"); setLocalLogout(true); setCentralLogout(true); @@ -29,7 +26,7 @@ public CentralLogoutController(Config config) { /** logout() method should not be called if oidc is not enabled */ public Result executeLogout(Http.Request request) { - if (_isOidcEnabled) { + if (ssoManager.isSsoEnabled()) { try { return logout(request).toCompletableFuture().get().withNewSession(); } catch (Exception e) { diff --git a/datahub-frontend/app/controllers/RedirectController.java b/datahub-frontend/app/controllers/RedirectController.java index 17f86b7fbffae..a86584e24ca29 100644 --- a/datahub-frontend/app/controllers/RedirectController.java +++ b/datahub-frontend/app/controllers/RedirectController.java @@ -16,7 +16,10 @@ public Result favicon(Http.Request request) { if (config.getVisualConfig().getAssets().getFaviconUrl().startsWith("http")) { return permanentRedirect(config.getVisualConfig().getAssets().getFaviconUrl()); } else { - final String prefix = config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/") ? "/public" : "/public/"; + final String prefix = + config.getVisualConfig().getAssets().getFaviconUrl().startsWith("/") + ? "/public" + : "/public/"; return ok(Application.class.getResourceAsStream( prefix + config.getVisualConfig().getAssets().getFaviconUrl())) .as("image/x-icon"); diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 750886570bf40..385b02a56ba23 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -5,8 +5,8 @@ import auth.sso.SsoProvider; import auth.sso.oidc.OidcCallbackLogic; import client.AuthServiceClient; -import com.datahub.authentication.Authentication; import com.linkedin.entity.client.SystemEntityClient; +import io.datahubproject.metadata.context.OperationContext; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -16,16 +16,15 @@ import javax.annotation.Nonnull; import javax.inject.Inject; import javax.inject.Named; - -import io.datahubproject.metadata.context.OperationContext; import lombok.extern.slf4j.Slf4j; +import org.pac4j.core.adapter.FrameworkAdapter; import org.pac4j.core.client.Client; import org.pac4j.core.client.Clients; import org.pac4j.core.config.Config; +import org.pac4j.core.context.FrameworkParameters; import org.pac4j.core.engine.CallbackLogic; -import org.pac4j.core.http.adapter.HttpActionAdapter; import org.pac4j.play.CallbackController; -import org.pac4j.play.PlayWebContext; +import org.pac4j.play.context.PlayFrameworkParameters; import play.mvc.Http; import play.mvc.Result; import play.mvc.Results; @@ -40,8 +39,9 @@ @Slf4j public class SsoCallbackController extends CallbackController { - private final SsoManager _ssoManager; - private final Config _config; + private final SsoManager ssoManager; + private final Config config; + private final CallbackLogic callbackLogic; @Inject public SsoCallbackController( @@ -51,23 +51,41 @@ public SsoCallbackController( @Nonnull AuthServiceClient authClient, @Nonnull Config config, @Nonnull com.typesafe.config.Config configs) { - _ssoManager = ssoManager; - _config = config; + this.ssoManager = ssoManager; + this.config = config; setDefaultUrl("/"); // By default, redirects to Home Page on log in. - setSaveInSession(false); - setCallbackLogic( + + callbackLogic = new SsoCallbackLogic( ssoManager, - systemOperationContext, + systemOperationContext, entityClient, authClient, - new CookieConfigs(configs))); + new CookieConfigs(configs)); + } + + @Override + public CompletionStage callback(Http.Request request) { + FrameworkAdapter.INSTANCE.applyDefaultSettingsIfUndefined(this.config); + + return CompletableFuture.supplyAsync( + () -> { + return (Result) + callbackLogic.perform( + this.config, + getDefaultUrl(), + getRenewSession(), + getDefaultClient(), + new PlayFrameworkParameters(request)); + }, + this.ec.current()); } public CompletionStage handleCallback(String protocol, Http.Request request) { if (shouldHandleCallback(protocol)) { - log.debug("Handling SSO callback. Protocol: {}", - _ssoManager.getSsoProvider().protocol().getCommonName()); + log.debug( + "Handling SSO callback. Protocol: {}", + ssoManager.getSsoProvider().protocol().getCommonName()); return callback(request) .handle( (res, e) -> { @@ -94,9 +112,9 @@ public CompletionStage handleCallback(String protocol, Http.Request requ } /** Logic responsible for delegating to protocol-specific callback logic. */ - public class SsoCallbackLogic implements CallbackLogic { + public class SsoCallbackLogic implements CallbackLogic { - private final OidcCallbackLogic _oidcCallbackLogic; + private final OidcCallbackLogic oidcCallbackLogic; SsoCallbackLogic( final SsoManager ssoManager, @@ -104,31 +122,21 @@ public class SsoCallbackLogic implements CallbackLogic { final SystemEntityClient entityClient, final AuthServiceClient authClient, final CookieConfigs cookieConfigs) { - _oidcCallbackLogic = + oidcCallbackLogic = new OidcCallbackLogic( ssoManager, systemOperationContext, entityClient, authClient, cookieConfigs); } @Override - public Result perform( - PlayWebContext context, + public Object perform( Config config, - HttpActionAdapter httpActionAdapter, - String defaultUrl, - Boolean saveInSession, - Boolean multiProfile, - Boolean renewSession, - String defaultClient) { - if (SsoProvider.SsoProtocol.OIDC.equals(_ssoManager.getSsoProvider().protocol())) { - return _oidcCallbackLogic.perform( - context, - config, - httpActionAdapter, - defaultUrl, - saveInSession, - multiProfile, - renewSession, - defaultClient); + String inputDefaultUrl, + Boolean inputRenewSession, + String defaultClient, + FrameworkParameters parameters) { + if (SsoProvider.SsoProtocol.OIDC.equals(ssoManager.getSsoProvider().protocol())) { + return oidcCallbackLogic.perform( + config, inputDefaultUrl, inputRenewSession, defaultClient, parameters); } // Should never occur. throw new UnsupportedOperationException( @@ -137,18 +145,18 @@ public Result perform( } private boolean shouldHandleCallback(final String protocol) { - if (!_ssoManager.isSsoEnabled()) { + if (!ssoManager.isSsoEnabled()) { return false; } updateConfig(); - return _ssoManager.getSsoProvider().protocol().getCommonName().equals(protocol); + return ssoManager.getSsoProvider().protocol().getCommonName().equals(protocol); } private void updateConfig() { final Clients clients = new Clients(); final List clientList = new ArrayList<>(); - clientList.add(_ssoManager.getSsoProvider().client()); + clientList.add(ssoManager.getSsoProvider().client()); clients.setClients(clientList); - _config.setClients(clients); + config.setClients(clients); } } diff --git a/datahub-frontend/app/controllers/TrackingController.java b/datahub-frontend/app/controllers/TrackingController.java index 254a8cc640d0c..5d12c96ed77cb 100644 --- a/datahub-frontend/app/controllers/TrackingController.java +++ b/datahub-frontend/app/controllers/TrackingController.java @@ -22,23 +22,23 @@ @Singleton public class TrackingController extends Controller { - private final Logger _logger = LoggerFactory.getLogger(TrackingController.class.getName()); + private static final Logger logger = LoggerFactory.getLogger(TrackingController.class.getName()); - private final String _topic; + private final String topic; - @Inject KafkaTrackingProducer _producer; + @Inject KafkaTrackingProducer producer; - @Inject AuthServiceClient _authClient; + @Inject AuthServiceClient authClient; @Inject public TrackingController(@Nonnull Config config) { - _topic = config.getString("analytics.tracking.topic"); + topic = config.getString("analytics.tracking.topic"); } @Security.Authenticated(Authenticator.class) @Nonnull public Result track(Http.Request request) throws Exception { - if (!_producer.isEnabled()) { + if (!producer.isEnabled()) { // If tracking is disabled, simply return a 200. return status(200); } @@ -51,15 +51,15 @@ public Result track(Http.Request request) throws Exception { } final String actor = request.session().data().get(ACTOR); try { - _logger.debug( + logger.debug( String.format("Emitting product analytics event. actor: %s, event: %s", actor, event)); final ProducerRecord record = - new ProducerRecord<>(_topic, actor, event.toString()); - _producer.send(record); - _authClient.track(event.toString()); + new ProducerRecord<>(topic, actor, event.toString()); + producer.send(record); + authClient.track(event.toString()); return ok(); } catch (Exception e) { - _logger.error( + logger.error( String.format( "Failed to emit product analytics event. actor: %s, event: %s", actor, event)); return internalServerError(e.getMessage()); diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle index ab4ce405a5541..7750e169b11fb 100644 --- a/datahub-frontend/build.gradle +++ b/datahub-frontend/build.gradle @@ -12,6 +12,12 @@ ext { docker_dir = 'datahub-frontend' } +java { + toolchain { + languageVersion = JavaLanguageVersion.of(jdkVersion(project)) + } +} + model { // Must specify the dependency here as "stage" is added by rule based model. tasks.myTar { diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index be57a33b13564..db982b595e248 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -184,6 +184,7 @@ auth.oidc.responseMode = ${?AUTH_OIDC_RESPONSE_MODE} auth.oidc.useNonce = ${?AUTH_OIDC_USE_NONCE} auth.oidc.customParam.resource = ${?AUTH_OIDC_CUSTOM_PARAM_RESOURCE} auth.oidc.readTimeout = ${?AUTH_OIDC_READ_TIMEOUT} +auth.oidc.connectTimeout = ${?AUTH_OIDC_CONNECT_TIMEOUT} auth.oidc.extractJwtAccessTokenClaims = ${?AUTH_OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS} # Whether to extract claims from JWT access token. Defaults to false. auth.oidc.preferredJwsAlgorithm = ${?AUTH_OIDC_PREFERRED_JWS_ALGORITHM} # Which jws algorithm to use auth.oidc.acrValues = ${?AUTH_OIDC_ACR_VALUES} diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index ff43e4a93a80f..266962721a80a 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -11,16 +11,29 @@ configurations { play } +ext { + nimbusJoseJwtVersion = "9.41.2" + oauth2OidcSdkVersion = "11.20.1" +} + dependencies { implementation project(':datahub-web-react') constraints { + play(externalDependency.pac4j) + play(externalDependency.playPac4j) + play("com.nimbusds:oauth2-oidc-sdk:$oauth2OidcSdkVersion") + play("com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion") + implementation(externalDependency.pac4j) + implementation(externalDependency.playPac4j) + implementation("com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion") + testImplementation("com.nimbusds:oauth2-oidc-sdk:$oauth2OidcSdkVersion") + play(externalDependency.jacksonDataBind) - play('com.nimbusds:oauth2-oidc-sdk:8.36.2') - play('com.nimbusds:nimbus-jose-jwt:8.18') - play('com.typesafe.akka:akka-actor_2.12:2.6.20') + play("com.typesafe.akka:akka-actor_$playScalaVersion:2.6.20") play(externalDependency.jsonSmart) play('io.netty:netty-all:4.1.114.Final') + implementation(externalDependency.commonsText) { because("previous versions are vulnerable to CVE-2022-42889") } @@ -46,14 +59,11 @@ dependencies { implementation externalDependency.jerseyCore implementation externalDependency.jerseyGuava - implementation(externalDependency.pac4j) { - exclude group: "net.minidev", module: "json-smart" - exclude group: "com.nimbusds", module: "nimbus-jose-jwt" - } - - implementation 'com.nimbusds:nimbus-jose-jwt:8.18' - implementation externalDependency.jsonSmart + implementation externalDependency.pac4j implementation externalDependency.playPac4j + implementation "com.nimbusds:nimbus-jose-jwt:$nimbusJoseJwtVersion" + implementation externalDependency.jsonSmart + implementation externalDependency.shiroCore implementation externalDependency.playCache @@ -69,7 +79,7 @@ dependencies { testImplementation externalDependency.mockito testImplementation externalDependency.playTest testImplementation 'org.awaitility:awaitility:4.2.0' - testImplementation 'no.nav.security:mock-oauth2-server:0.3.1' + testImplementation 'no.nav.security:mock-oauth2-server:2.1.9' testImplementation 'org.junit-pioneer:junit-pioneer:1.9.1' testImplementation externalDependency.junitJupiterApi testRuntimeOnly externalDependency.junitJupiterEngine @@ -78,7 +88,7 @@ dependencies { compileOnly externalDependency.lombok runtimeOnly externalDependency.guicePlay runtimeOnly (externalDependency.playDocs) { - exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12' + exclude group: 'com.typesafe.akka', module: "akka-http-core_$playScalaVersion" } runtimeOnly externalDependency.playGuice implementation externalDependency.log4j2Api @@ -90,9 +100,9 @@ dependencies { play { platform { - playVersion = '2.8.21' - scalaVersion = '2.12' - javaVersion = JavaVersion.VERSION_11 + playVersion = "2.8.22" // see also top level build.gradle + scalaVersion = "2.13" + javaVersion = JavaVersion.VERSION_17 } injectedRoutesGenerator = true diff --git a/datahub-frontend/test/app/ApplicationTest.java b/datahub-frontend/test/app/ApplicationTest.java index 534cffb5cc7fe..3ad9e22857168 100644 --- a/datahub-frontend/test/app/ApplicationTest.java +++ b/datahub-frontend/test/app/ApplicationTest.java @@ -12,17 +12,27 @@ import com.nimbusds.jwt.JWTParser; import controllers.routes; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.InetAddress; +import java.net.URL; import java.text.ParseException; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import no.nav.security.mock.oauth2.MockOAuth2Server; +import no.nav.security.mock.oauth2.http.OAuth2HttpRequest; +import no.nav.security.mock.oauth2.http.OAuth2HttpResponse; +import no.nav.security.mock.oauth2.http.Route; import no.nav.security.mock.oauth2.token.DefaultOAuth2TokenCallback; +import okhttp3.Headers; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import org.awaitility.Awaitility; import org.awaitility.Durations; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -30,6 +40,8 @@ import org.junitpioneer.jupiter.SetEnvironmentVariable; import org.openqa.selenium.Cookie; import org.openqa.selenium.htmlunit.HtmlUnitDriver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import play.Application; import play.Environment; import play.Mode; @@ -48,7 +60,9 @@ @SetEnvironmentVariable(key = "AUTH_OIDC_JIT_PROVISIONING_ENABLED", value = "false") @SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_ID", value = "testclient") @SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_SECRET", value = "testsecret") +@SetEnvironmentVariable(key = "AUTH_VERBOSE_LOGGING", value = "true") public class ApplicationTest extends WithBrowser { + private static final Logger logger = LoggerFactory.getLogger(ApplicationTest.class); private static final String ISSUER_ID = "testIssuer"; @Override @@ -80,41 +94,34 @@ public int gmsServerPort() { return providePort() + 2; } - private MockOAuth2Server _oauthServer; - private MockWebServer _gmsServer; + private MockOAuth2Server oauthServer; + private Thread oauthServerThread; + private CompletableFuture oauthServerStarted; - private String _wellKnownUrl; + private MockWebServer gmsServer; + + private String wellKnownUrl; private static final String TEST_USER = "urn:li:corpuser:testUser@myCompany.com"; private static final String TEST_TOKEN = "faketoken_YCpYIrjQH4sD3_rAc3VPPFg4"; @BeforeAll public void init() throws IOException { - _gmsServer = new MockWebServer(); - _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested - _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested - _gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested - _gmsServer.enqueue(new MockResponse().setBody(String.format("{\"value\":\"%s\"}", TEST_USER))); - _gmsServer.enqueue( + // Start Mock GMS + gmsServer = new MockWebServer(); + gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested + gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested + gmsServer.enqueue(new MockResponse().setResponseCode(404)); // dynamic settings - not tested + gmsServer.enqueue(new MockResponse().setBody(String.format("{\"value\":\"%s\"}", TEST_USER))); + gmsServer.enqueue( new MockResponse().setBody(String.format("{\"accessToken\":\"%s\"}", TEST_TOKEN))); - _gmsServer.start(gmsServerPort()); - - _oauthServer = new MockOAuth2Server(); - _oauthServer.enqueueCallback( - new DefaultOAuth2TokenCallback( - ISSUER_ID, - "testUser", - List.of(), - Map.of( - "email", "testUser@myCompany.com", - "groups", "myGroup"), - 600)); - _oauthServer.start(InetAddress.getByName("localhost"), oauthServerPort()); - - // Discovery url to authorization server metadata - _wellKnownUrl = _oauthServer.wellKnownUrl(ISSUER_ID).toString(); + gmsServer.start(gmsServerPort()); + // Start Mock Identity Provider + startMockOauthServer(); + // Start Play Frontend startServer(); + // Start Browser createBrowser(); Awaitility.await().timeout(Durations.TEN_SECONDS).until(() -> app != null); @@ -122,13 +129,131 @@ public void init() throws IOException { @AfterAll public void shutdown() throws IOException { - if (_gmsServer != null) { - _gmsServer.shutdown(); - } - if (_oauthServer != null) { - _oauthServer.shutdown(); + if (gmsServer != null) { + logger.info("Shutdown Mock GMS"); + gmsServer.shutdown(); } + logger.info("Shutdown Play Frontend"); stopServer(); + if (oauthServer != null) { + logger.info("Shutdown MockOAuth2Server"); + oauthServer.shutdown(); + } + if (oauthServerThread != null && oauthServerThread.isAlive()) { + logger.info("Shutdown MockOAuth2Server thread"); + oauthServerThread.interrupt(); + try { + oauthServerThread.join(2000); // Wait up to 2 seconds for thread to finish + } catch (InterruptedException e) { + logger.warn("Shutdown MockOAuth2Server thread failed to join."); + } + } + } + + private void startMockOauthServer() { + // Configure HEAD responses + Route[] routes = + new Route[] { + new Route() { + @Override + public boolean match(@NotNull OAuth2HttpRequest oAuth2HttpRequest) { + return "HEAD".equals(oAuth2HttpRequest.getMethod()) + && (String.format("/%s/.well-known/openid-configuration", ISSUER_ID) + .equals(oAuth2HttpRequest.getUrl().url().getPath()) + || String.format("/%s/token", ISSUER_ID) + .equals(oAuth2HttpRequest.getUrl().url().getPath())); + } + + @Override + public OAuth2HttpResponse invoke(OAuth2HttpRequest oAuth2HttpRequest) { + return new OAuth2HttpResponse( + Headers.of( + Map.of( + "Content-Type", "application/json", + "Cache-Control", "no-store", + "Pragma", "no-cache", + "Content-Length", "-1")), + 200, + null, + null); + } + } + }; + oauthServer = new MockOAuth2Server(routes); + oauthServerStarted = new CompletableFuture<>(); + + // Create and start server in separate thread + oauthServerThread = + new Thread( + () -> { + try { + // Configure mock responses + oauthServer.enqueueCallback( + new DefaultOAuth2TokenCallback( + ISSUER_ID, + "testUser", + "JWT", + List.of(), + Map.of( + "email", "testUser@myCompany.com", + "groups", "myGroup"), + 600)); + + oauthServer.start(InetAddress.getByName("localhost"), oauthServerPort()); + + oauthServerStarted.complete(null); + + // Keep thread alive until server is stopped + while (!Thread.currentThread().isInterrupted() && testServer.isRunning()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } catch (Exception e) { + oauthServerStarted.completeExceptionally(e); + } + }); + + oauthServerThread.setDaemon(true); // Ensure thread doesn't prevent JVM shutdown + oauthServerThread.start(); + + // Wait for server to start with timeout + oauthServerStarted + .orTimeout(10, TimeUnit.SECONDS) + .whenComplete( + (result, throwable) -> { + if (throwable != null) { + if (throwable instanceof TimeoutException) { + throw new RuntimeException( + "MockOAuth2Server failed to start within timeout", throwable); + } + throw new RuntimeException("MockOAuth2Server failed to start", throwable); + } + }); + + // Discovery url to authorization server metadata + wellKnownUrl = oauthServer.wellKnownUrl(ISSUER_ID).toString(); + + // Wait for server to return configuration + // Validate mock server returns data + try { + URL url = new URL(wellKnownUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + int responseCode = conn.getResponseCode(); + logger.info("Well-known endpoint response code: {}", responseCode); + + if (responseCode != 200) { + throw new RuntimeException( + "MockOAuth2Server not accessible. Response code: " + responseCode); + } + logger.info("Successfully started MockOAuth2Server."); + } catch (Exception e) { + throw new RuntimeException("Failed to connect to MockOAuth2Server", e); + } } @Test @@ -158,7 +283,7 @@ public void testIndexNotFound() { public void testOpenIdConfig() { assertEquals( "http://localhost:" + oauthServerPort() + "/testIssuer/.well-known/openid-configuration", - _wellKnownUrl); + wellKnownUrl); } @Test @@ -188,10 +313,10 @@ public void testHappyPathOidc() throws ParseException { @Test public void testAPI() throws ParseException { testHappyPathOidc(); - int requestCount = _gmsServer.getRequestCount(); + int requestCount = gmsServer.getRequestCount(); browser.goTo("/api/v2/graphql/"); - assertEquals(++requestCount, _gmsServer.getRequestCount()); + assertEquals(++requestCount, gmsServer.getRequestCount()); } @Test @@ -201,8 +326,9 @@ public void testOidcRedirectToRequestedUrl() { } /** - * The Redirect Uri parameter is used to store a previous relative location within the app to be able to - * take a user back to their expected page. Redirecting to other domains should be blocked. + * The Redirect Uri parameter is used to store a previous relative location within the app to be + * able to take a user back to their expected page. Redirecting to other domains should be + * blocked. */ @Test public void testInvalidRedirectUrl() { diff --git a/datahub-frontend/test/oidc/OidcCallbackLogicTest.java b/datahub-frontend/test/oidc/OidcCallbackLogicTest.java index f4784c29e91f2..9eb3833cbc897 100644 --- a/datahub-frontend/test/oidc/OidcCallbackLogicTest.java +++ b/datahub-frontend/test/oidc/OidcCallbackLogicTest.java @@ -1,64 +1,65 @@ package oidc; -import auth.sso.oidc.OidcConfigs; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - import static auth.sso.oidc.OidcCallbackLogic.getGroupNames; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Arrays; +import java.util.Collection; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; import org.pac4j.core.profile.CommonProfile; public class OidcCallbackLogicTest { - @Test - public void testGetGroupsClaimNamesJsonArray() { - CommonProfile profile = createMockProfileWithAttribute("[\"group1\", \"group2\"]", "groupsClaimName"); - Collection result = getGroupNames(profile, "[\"group1\", \"group2\"]", "groupsClaimName"); - assertEquals(Arrays.asList("group1", "group2"), result); - } - @Test - public void testGetGroupNamesWithSingleGroup() { - CommonProfile profile = createMockProfileWithAttribute("group1", "groupsClaimName"); - Collection result = getGroupNames(profile, "group1", "groupsClaimName"); - assertEquals(Arrays.asList("group1"), result); - } + @Test + public void testGetGroupsClaimNamesJsonArray() { + CommonProfile profile = + createMockProfileWithAttribute("[\"group1\", \"group2\"]", "groupsClaimName"); + Collection result = + getGroupNames(profile, "[\"group1\", \"group2\"]", "groupsClaimName"); + assertEquals(Arrays.asList("group1", "group2"), result); + } - @Test - public void testGetGroupNamesWithCommaSeparated() { - CommonProfile profile = createMockProfileWithAttribute("group1,group2", "groupsClaimName"); - Collection result = getGroupNames(profile, "group1,group2", "groupsClaimName"); - assertEquals(Arrays.asList("group1", "group2"), result); - } + @Test + public void testGetGroupNamesWithSingleGroup() { + CommonProfile profile = createMockProfileWithAttribute("group1", "groupsClaimName"); + Collection result = getGroupNames(profile, "group1", "groupsClaimName"); + assertEquals(Arrays.asList("group1"), result); + } - @Test - public void testGetGroupNamesWithCollection() { - CommonProfile profile = createMockProfileWithAttribute(Arrays.asList("group1", "group2"), "groupsClaimName"); - Collection result = getGroupNames(profile, Arrays.asList("group1", "group2"), "groupsClaimName"); - assertEquals(Arrays.asList("group1", "group2"), result); - } - // Helper method to create a mock CommonProfile with given attribute - private CommonProfile createMockProfileWithAttribute(Object attribute, String attributeName) { - CommonProfile profile = mock(CommonProfile.class); + @Test + public void testGetGroupNamesWithCommaSeparated() { + CommonProfile profile = createMockProfileWithAttribute("group1,group2", "groupsClaimName"); + Collection result = getGroupNames(profile, "group1,group2", "groupsClaimName"); + assertEquals(Arrays.asList("group1", "group2"), result); + } + + @Test + public void testGetGroupNamesWithCollection() { + CommonProfile profile = + createMockProfileWithAttribute(Arrays.asList("group1", "group2"), "groupsClaimName"); + Collection result = + getGroupNames(profile, Arrays.asList("group1", "group2"), "groupsClaimName"); + assertEquals(Arrays.asList("group1", "group2"), result); + } - // Mock for getAttribute(String) - when(profile.getAttribute(attributeName)).thenReturn(attribute); + // Helper method to create a mock CommonProfile with given attribute + private CommonProfile createMockProfileWithAttribute(Object attribute, String attributeName) { + CommonProfile profile = mock(CommonProfile.class); - // Mock for getAttribute(String, Class) - if (attribute instanceof Collection) { - when(profile.getAttribute(attributeName, Collection.class)).thenReturn((Collection) attribute); - } else if (attribute instanceof String) { - when(profile.getAttribute(attributeName, String.class)).thenReturn((String) attribute); - } - // Add more conditions here if needed for other types + // Mock for getAttribute(String) + when(profile.getAttribute(attributeName)).thenReturn(attribute); - return profile; + // Mock for getAttribute(String, Class) + if (attribute instanceof Collection) { + when(profile.getAttribute(attributeName, Collection.class)) + .thenReturn((Collection) attribute); + } else if (attribute instanceof String) { + when(profile.getAttribute(attributeName, String.class)).thenReturn((String) attribute); } + // Add more conditions here if needed for other types + + return profile; + } } diff --git a/datahub-frontend/test/security/OidcConfigurationTest.java b/datahub-frontend/test/security/OidcConfigurationTest.java index 1c52d45af5f9e..ec19979c56120 100644 --- a/datahub-frontend/test/security/OidcConfigurationTest.java +++ b/datahub-frontend/test/security/OidcConfigurationTest.java @@ -23,9 +23,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.pac4j.oidc.client.OidcClient; -import org.json.JSONObject; public class OidcConfigurationTest { @@ -328,17 +328,28 @@ public void readPreferredJwsAlgorithmPropagationFromConfig() { oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR); OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder); OidcProvider oidcProvider = new OidcProvider(oidcConfigs); - assertEquals("RS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString()); + assertEquals( + "RS256", + ((OidcClient) oidcProvider.client()) + .getConfiguration() + .getPreferredJwsAlgorithm() + .toString()); } @Test public void readPreferredJwsAlgorithmPropagationFromJSON() { - final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString(); + final String SSO_SETTINGS_JSON_STR = + new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString(); CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256")); OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder(); oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR); OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder); OidcProvider oidcProvider = new OidcProvider(oidcConfigs); - assertEquals("HS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString()); + assertEquals( + "HS256", + ((OidcClient) oidcProvider.client()) + .getConfiguration() + .getPreferredJwsAlgorithm() + .toString()); } } From a8af347588686aa52fe36d473be1b2e6be25dd52 Mon Sep 17 00:00:00 2001 From: kartikey-visa <164213341+kartikey-visa@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:43:40 +0530 Subject: [PATCH 20/26] fix: business attribute empty bubble selection issue (#11720) --- .../shared/businessAttribute/AddBusinessAttributeModal.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx index 026cf6f6fa724..ea51ac4e12c3f 100644 --- a/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx +++ b/datahub-web-react/src/app/shared/businessAttribute/AddBusinessAttributeModal.tsx @@ -155,7 +155,9 @@ export default function EditBusinessAttributeModal({ event.stopPropagation(); }; /* eslint-disable-next-line react/prop-types */ - const selectedItem = displayedAttributes.find((attribute) => attribute.urn === value)?.component; + const selectedItem = selectedAttribute + ? selectedAttribute?.props?.component + : displayedAttributes.find((attribute) => attribute.urn === value)?.component; return ( {selectedItem} @@ -172,7 +174,7 @@ export default function EditBusinessAttributeModal({ const onSelectValue = (selectedUrn: string) => { const selectedSearchOption = attributeSearchOptions?.find((option) => option.props.value === selectedUrn); setUrn(selectedUrn); - if (!selectedAttribute) { + if (selectedAttribute?.selectedUrn !== selectedUrn) { setSelectedAttribute({ selectedUrn, component: ( From 25a4eefe6bf5436afbaaeff1ee22b9e62ade5a16 Mon Sep 17 00:00:00 2001 From: kartikey-visa <164213341+kartikey-visa@users.noreply.github.com> Date: Mon, 28 Oct 2024 23:59:07 +0530 Subject: [PATCH 21/26] feat(frontend): show browse paths for business attributes related entities - schema fields (#11585) --- .../SchemaFieldPropertiesEntity.tsx | 33 +++++++++++-- .../entity/schemaField/preview/Preview.tsx | 17 ++++++- .../header/PlatformContent/DatasetLink.tsx | 46 +++++++++++++++++++ .../PlatformContent/PlatformContentView.tsx | 11 ++++- .../src/app/preview/DefaultPreviewCard.tsx | 4 ++ datahub-web-react/src/graphql/search.graphql | 6 +++ 6 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx diff --git a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx index 7e74b43e68afb..88743012ddbc8 100644 --- a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx +++ b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx @@ -1,9 +1,10 @@ import * as React from 'react'; import { PicCenterOutlined } from '@ant-design/icons'; -import { EntityType, SchemaFieldEntity, SearchResult } from '../../../types.generated'; +import { Dataset, EntityType, SchemaFieldEntity, SearchResult } from '../../../types.generated'; import { Entity, IconStyleType, PreviewType } from '../Entity'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { Preview } from './preview/Preview'; +import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; export class SchemaFieldPropertiesEntity implements Entity { type: EntityType = EntityType.SchemaField; @@ -18,6 +19,16 @@ export class SchemaFieldPropertiesEntity implements Entity { isLineageEnabled = () => false; + getParentDataset = (parent) => { + return { + urn: parent?.urn, + name: parent?.name, + type: parent?.type, + platform: parent?.platfrom, + properties: parent?.properties, + } as Dataset; + }; + // Currently unused. getAutoCompleteFieldName = () => 'schemaField'; @@ -33,9 +44,23 @@ export class SchemaFieldPropertiesEntity implements Entity { // Currently unused. renderProfile = (_: string) => <>; - renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => ( - - ); + renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => { + const parent = data.parent as Dataset; + return ( + + ); + }; renderSearch = (result: SearchResult) => this.renderPreview(PreviewType.SEARCH, result.entity as SchemaFieldEntity); diff --git a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx index b22e988c76672..086b1a3802188 100644 --- a/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx +++ b/datahub-web-react/src/app/entity/schemaField/preview/Preview.tsx @@ -1,6 +1,6 @@ import React from 'react'; import { PicCenterOutlined } from '@ant-design/icons'; -import { EntityType, Owner } from '../../../../types.generated'; +import { Dataset, EntityType, Owner, ParentContainersResult } from '../../../../types.generated'; import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType, PreviewType } from '../../Entity'; @@ -11,12 +11,22 @@ export const Preview = ({ description, owners, previewType, + parentContainers, + platformName, + platformLogo, + platformInstanceId, + parentDataset, }: { datasetUrn: string; name: string; description?: string | null; owners?: Array | null; previewType: PreviewType; + parentContainers?: ParentContainersResult | null; + platformName?: string; + platformLogo?: string | null; + platformInstanceId?: string; + parentDataset?: Dataset; }): JSX.Element => { const entityRegistry = useEntityRegistry(); @@ -35,6 +45,11 @@ export const Preview = ({ logoComponent={} type="Column" typeIcon={entityRegistry.getIcon(EntityType.SchemaField, 14, IconStyleType.ACCENT)} + logoUrl={platformLogo || ''} + platform={platformName} + platformInstanceId={platformInstanceId} + parentContainers={parentContainers} + parentDataset={parentDataset} /> ); }; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx new file mode 100644 index 0000000000000..cf4891acd384d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/DatasetLink.tsx @@ -0,0 +1,46 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Link } from 'react-router-dom'; +import { Typography } from 'antd'; +import { Dataset, EntityType } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { useEntityRegistry } from '../../../../../../useEntityRegistry'; +import { IconStyleType } from '../../../../../Entity'; + +const DatasetText = styled(Typography.Text)` + font-size: 12px; + line-height: 20px; + color: ${ANTD_GRAY[7]}; +`; + +export const DatasetIcon = styled.span` + color: ${ANTD_GRAY[7]}; + margin-right: 4px; + font-size: 12px; +`; + +const StyledLink = styled(Link)` + white-space: nowrap; +`; + +interface Props { + parentDataset: Dataset; +} + +function DatasetLink(props: Props) { + const { parentDataset } = props; + const entityRegistry = useEntityRegistry(); + if (!parentDataset) return null; + + const datasetUrl = entityRegistry.getEntityUrl(EntityType.Dataset, parentDataset.urn); + const datasetName = entityRegistry.getDisplayName(EntityType.Dataset, parentDataset); + + return ( + + {entityRegistry.getIcon(EntityType.Dataset, 14, IconStyleType.ACCENT)} + {datasetName} + + ); +} + +export default DatasetLink; diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx index 3104dfd911aab..70cc5fa3b36d1 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/PlatformContent/PlatformContentView.tsx @@ -2,9 +2,10 @@ import React from 'react'; import styled from 'styled-components'; import { Typography, Image } from 'antd'; import { Maybe } from 'graphql/jsutils/Maybe'; -import { Container, Entity } from '../../../../../../../types.generated'; +import { Container, Dataset, Entity } from '../../../../../../../types.generated'; import { ANTD_GRAY } from '../../../../constants'; import ContainerLink from './ContainerLink'; +import DatasetLink from './DatasetLink'; import { StyledRightOutlined, ParentNodesWrapper as ParentContainersWrapper, @@ -80,6 +81,7 @@ interface Props { parentEntities?: Entity[] | null; parentContainersRef: React.RefObject; areContainersTruncated: boolean; + parentDataset?: Dataset; } function PlatformContentView(props: Props) { @@ -96,6 +98,7 @@ function PlatformContentView(props: Props) { parentContainers, parentContainersRef, areContainersTruncated, + parentDataset, } = props; const directParentContainer = parentContainers && parentContainers[0]; @@ -152,6 +155,12 @@ function PlatformContentView(props: Props) { {directParentContainer && } + {parentDataset && ( + + + + + )} ); diff --git a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx index ee29e92e06a22..0d5cfaaf42b9a 100644 --- a/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx +++ b/datahub-web-react/src/app/preview/DefaultPreviewCard.tsx @@ -9,6 +9,7 @@ import { GlossaryTerms, SearchInsight, Container, + Dataset, ParentContainersResult, Maybe, CorpUser, @@ -196,6 +197,7 @@ interface Props { previewType?: Maybe; paths?: EntityPath[]; health?: Health[]; + parentDataset?: Dataset; } export default function DefaultPreviewCard({ @@ -238,6 +240,7 @@ export default function DefaultPreviewCard({ previewType, paths, health, + parentDataset, }: Props) { // sometimes these lists will be rendered inside an entity container (for example, in the case of impact analysis) // in those cases, we may want to enrich the preview w/ context about the container entity @@ -284,6 +287,7 @@ export default function DefaultPreviewCard({ parentEntities={parentEntities} parentContainersRef={contentRef} areContainersTruncated={isContentTruncated} + parentDataset={parentDataset} /> diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 38c9df0a636d0..4a10d5fe250de 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -980,6 +980,12 @@ fragment entityField on SchemaFieldEntity { platform { ...platformFields } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + parentContainers { + ...parentContainersFields + } } } fieldPath From e90d744cbdc01e92f7f50907a23306fab12a961e Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 29 Oct 2024 02:46:34 +0530 Subject: [PATCH 22/26] feat(bigquery): support config for region qualifiers to fetch jobs (#11728) --- .../src/datahub/ingestion/source/bigquery_v2/bigquery.py | 1 + .../datahub/ingestion/source/bigquery_v2/bigquery_config.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 0e986acc81add..a1bbb9dd6b0b9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -282,6 +282,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: include_usage_statistics=self.config.include_usage_statistics, include_operations=self.config.usage.include_operational_stats, top_n_queries=self.config.usage.top_n_queries, + region_qualifiers=self.config.region_qualifiers, ), structured_report=self.report, filters=self.filters, diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index 1f74fffeab7fb..331b583423093 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -527,6 +527,12 @@ def have_table_data_read_permission(self) -> bool: " Set to 1 to disable.", ) + region_qualifiers: List[str] = Field( + default=["region-us", "region-eu"], + description="BigQuery regions to be scanned for bigquery jobs when using `use_queries_v2`. " + "See [this](https://cloud.google.com/bigquery/docs/information-schema-jobs#scope_and_syntax) for details.", + ) + # include_view_lineage and include_view_column_lineage are inherited from SQLCommonConfig # but not used in bigquery so we hide them from docs. include_view_lineage: bool = Field(default=True, hidden_from_docs=True) From 99d2de218c72982c04d28e7da912c4c9794182f4 Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:27:38 -0400 Subject: [PATCH 23/26] feat(docs) steps on handling workspace admin approvals for slack installation (#11726) --- docs/managed-datahub/slack/saas-slack-setup.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/managed-datahub/slack/saas-slack-setup.md b/docs/managed-datahub/slack/saas-slack-setup.md index 590f57d2c2e70..f242d40203d08 100644 --- a/docs/managed-datahub/slack/saas-slack-setup.md +++ b/docs/managed-datahub/slack/saas-slack-setup.md @@ -29,7 +29,11 @@ The following steps should be performed by a Slack Workspace Admin.

-6. You will be automatically re-directed to Slack to confirm DataHub Slack App's permissions and complete the installation process: +6. You will be automatically re-directed to Slack to confirm DataHub Slack App's permissions and complete the installation process. + +:::note +You may need approval from a workspace admin to do this step. Learn about [what to do in this scenario here](#workspace-admin-approval-guide). +:::

@@ -44,6 +48,14 @@ The following steps should be performed by a Slack Workspace Admin. Now proceed to the [Subscriptions and Notifications page](https://datahubproject.io/docs/managed-datahub/subscription-and-notification) to see how you can subscribe to be notified about events on the platform, or visit the [Slack App page](saas-slack-app.md) to see how you can use DataHub's powerful capabilities directly within Slack. +### Workspace admin approval guide +In some workspaces, you will find at step 6 above you will need approval from your workspace admin. In this case, you will want to: +1. On step #6 above, continue by clicking the button to request their approval. +2. Once approved, you will get a notification from the **Slack bot** that your request has been approved. Follow the link it provides to complete the process. +**If you do not get the message:** visit [api.slack.com/apps](https://api.slack.com/apps), open your DataHub app and complete the installation from there. +3. Once it is done, you can visit `/settings/notifications`, and send a test notification to verify everything's working. +4. **Making Slack commands work:** when running Slack commands such as `/datahub search my dataset` you may get a `dispatch_failed` error. To resolve this, see [the steps here](https://datahubproject.io/docs/managed-datahub/slack/saas-slack-troubleshoot#slack-bot-issues). +