feat(ingest/powerbi): fix subTypes and add workspace_type_filter (dat…

…ahub-project#11523) Co-authored-by: Harshal Sheth <[email protected]>
sleeperdeep · Dec 17, 2024 · d9523a9 · d9523a9
1 parent 3a4f608
commit d9523a9
Show file tree

Hide file tree

Showing 28 changed files with 2,870 additions and 716 deletions.
diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md
@@ -18,9 +18,11 @@
 | `Report.webUrl`       | `Chart.externalUrl` |
 | `Workspace`           | `Container`         |
 | `Report`              | `Dashboard`         |
+| `PaginatedReport`     | `Dashboard`         |
 | `Page`                | `Chart`             |
 
-If Tile is created from report then Chart.externalUrl is set to Report.webUrl.
+- If `Tile` is created from report then `Chart.externalUrl` is set to Report.webUrl.
+- The `Page` is unavailable for PowerBI PaginatedReport.
 
 ## Lineage
 

diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py
@@ -14,7 +14,6 @@ class DatasetSubTypes(StrEnum):
     ELASTIC_DATASTREAM = "Datastream"
     SALESFORCE_CUSTOM_OBJECT = "Custom Object"
     SALESFORCE_STANDARD_OBJECT = "Object"
-    POWERBI_DATASET_TABLE = "PowerBI Dataset Table"
     QLIK_DATASET = "Qlik Dataset"
     BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot"
     SHARDED_TABLE = "Sharded Table"
@@ -48,8 +47,8 @@ class BIContainerSubTypes(StrEnum):
     LOOKML_PROJECT = "LookML Project"
     LOOKML_MODEL = "LookML Model"
     TABLEAU_WORKBOOK = "Workbook"
-    POWERBI_WORKSPACE = "Workspace"
-    POWERBI_DATASET = "PowerBI Dataset"
+    POWERBI_DATASET = "Semantic Model"
+    POWERBI_DATASET_TABLE = "Table"
     QLIK_SPACE = "Qlik Space"
     QLIK_APP = "Qlik App"
     SIGMA_WORKSPACE = "Sigma Workspace"

diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
@@ -1,7 +1,7 @@
 import logging
 from dataclasses import dataclass, field as dataclass_field
 from enum import Enum
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Literal, Optional, Union
 
 import pydantic
 from pydantic import validator
@@ -47,6 +47,7 @@ class Constant:
     WORKSPACE_ID = "workspaceId"
     DASHBOARD_ID = "powerbi.linkedin.com/dashboards/{}"
     DATASET_EXECUTE_QUERIES = "DATASET_EXECUTE_QUERIES_POST"
+    GET_WORKSPACE_APP = "GET_WORKSPACE_APP"
     DATASET_ID = "datasetId"
     REPORT_ID = "reportId"
     SCAN_ID = "ScanId"
@@ -118,6 +119,15 @@ class Constant:
     CHART_COUNT = "chartCount"
     WORKSPACE_NAME = "workspaceName"
     DATASET_WEB_URL = "datasetWebUrl"
+    TYPE = "type"
+    REPORT_TYPE = "reportType"
+    LAST_UPDATE = "lastUpdate"
+    APP_ID = "appId"
+    REPORTS = "reports"
+    ORIGINAL_REPORT_OBJECT_ID = "originalReportObjectId"
+    APP_SUB_TYPE = "App"
+    STATE = "state"
+    ACTIVE = "Active"
 
 
 @dataclass
@@ -273,7 +283,8 @@ class PowerBiDashboardSourceConfig(
     # PowerBi workspace identifier
     workspace_id_pattern: AllowDenyPattern = pydantic.Field(
         default=AllowDenyPattern.allow_all(),
-        description="Regex patterns to filter PowerBI workspaces in ingestion",
+        description="Regex patterns to filter PowerBI workspaces in ingestion."
+        " Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces.",
     )
 
     # Dataset type mapping PowerBI support many type of data-sources. Here user need to define what type of PowerBI
@@ -340,7 +351,7 @@ class PowerBiDashboardSourceConfig(
     )
     modified_since: Optional[str] = pydantic.Field(
         default=None,
-        description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludePersonalWorkspaces and excludeInActiveWorkspaces limit to last 30 days",
+        description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludeInActiveWorkspaces limit to last 30 days",
     )
     extract_dashboards: bool = pydantic.Field(
         default=True,
@@ -445,6 +456,16 @@ class PowerBiDashboardSourceConfig(
         description="Patch dashboard metadata",
     )
 
+    workspace_type_filter: List[
+        Literal[
+            "Workspace", "PersonalGroup", "Personal", "AdminWorkspace", "AdminInsights"
+        ]
+    ] = pydantic.Field(
+        default=["Workspace"],
+        description="Ingest the metadata of the workspace where the workspace type corresponds to the specified workspace_type_filter."
+        " Note: This field works in conjunction with 'workspace_id_pattern'. Both must be matched for a workspace to be processed.",
+    )
+
     @root_validator(skip_on_failure=True)
     def validate_extract_column_level_lineage(cls, values: Dict) -> Dict:
         flags = [