Lilem/feat/fab a irchive/hackaton

Hackaton/Helper_.py

@@ -0,0 +1,38 @@
+import requests
+from msal import ConfidentialClientApplication
+
+
+class Helper_Token:
+    def __init__(
+        self,
+        tenant_id: str,
+        app_id: str
+    ) -> None: 
+
+        self.tenant_id = tenant_id
+        self.app_id = app_id   
+
+    def get_token_ServPrincRead_token(self, scope:str, secret: str) -> str:
+        authority = f'https://login.microsoftonline.com/{self.tenant_id}'
+        app = ConfidentialClientApplication(
+            client_id = self.app_id,
+            authority=authority,
+            client_credential=secret
+        )
+
+
+        token_response = app.acquire_token_for_client(scopes=[f'{scope}.default'])
+        #print(token_response)
+        access_token = token_response['access_token']
+
+        return access_token
+
+    def get_token_ServPrincRead(access_token:str) -> str:
+
+        headers = {
+            'Authorization': f'Bearer {access_token}',
+            'Content-Type': 'application/json',
+            'Accept': 'application/json'
+        }
+
+        return headers

Hackaton/ScanAPIs_Execution.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"markdown","id":"20e53853-65a0-40db-9b48-0f0600bc3ba0","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["#### Token Configuration\n"]},{"cell_type":"markdown","id":"9826e43d-c3c8-43e0-ac7a-4d893554c044","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["###### Variables"]},{"cell_type":"code","execution_count":null,"id":"afc18ee0-32d6-439d-a51a-77f2004292b9","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"outputs":[],"source":["# Replace these values with your actual authentication and resource details\n","tenant_id = \"\"\n","client_id = \"\"\n","client_secret = \"\" \n","\n","\n","\n"]},{"cell_type":"markdown","id":"122bb258-798f-49cd-bf0a-dd368eac90d9","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["### Getting Workspace metadata\n"]},{"cell_type":"markdown","id":"617454dd-f50e-4dd3-855b-f82e59055551","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["#### Fabric Token Scope"]},{"cell_type":"code","execution_count":2,"id":"a118999f-915e-4e75-8a48-94d841643755","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2024-09-18T19:48:17.5807431Z","execution_start_time":"2024-09-18T19:48:16.7290055Z","livy_statement_state":"available","normalized_state":"finished","parent_msg_id":"bfe7afe5-9cab-43c7-8e95-1419564baf14","queued_time":"2024-09-18T19:43:12.4495132Z","session_id":"6822d424-8d13-4523-9b71-8ee604dd4808","session_start_time":null,"spark_pool":null,"state":"finished","statement_id":6,"statement_ids":[6]},"text/plain":["StatementMeta(, 6822d424-8d13-4523-9b71-8ee604dd4808, 6, Finished, Available, Finished)"]},"metadata":{},"output_type":"display_data"}],"source":["from Helper_ import Helper_Token\n","\n","resource = 'https://api.fabric.microsoft.com/'\n","\n","Helper_=Helper_Token(tenant_id=tenant_id,app_id=client_id)\n","Token=Helper_.get_token_ServPrincRead_token(scope=resource,secret=client_secret)\n","headers_fabric=Helper_Token.get_token_ServPrincRead(Token)\n"]},{"cell_type":"markdown","id":"eef44bdb-6c0c-40fb-a407-278cdb7ed7f5","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["##### POwer BI Token Scope"]},{"cell_type":"code","execution_count":null,"id":"a063aab6-b90d-4a2b-8dc4-386073b8a71c","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["from Helper_ import Helper_Token\n","\n","resource = 'https://analysis.windows.net/powerbi/api/'\n","\n","\n","Helper_=Helper_Token(tenant_id=tenant_id,app_id=client_id)\n","Token=Helper_.get_token_ServPrincRead_token(scope=resource,secret=client_secret)\n","headers_powerbi=Helper_Token.get_token_ServPrincRead(Token)\n"]},{"cell_type":"markdown","id":"684fb850-873c-4ed9-b38c-25cf20f58243","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["### Executing the Class to get the Metadata information\n","#### Fabric API is used to list workspaces\n","#### Power BI is used to list the workspaces metadata information"]},{"cell_type":"code","execution_count":null,"id":"1519bf73-27a1-42cb-bebc-d5a6f351af9c","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["    def flatten_json(nested_json, parent_key='', sep='_'):\n","        \"\"\"\n","        Flatten a nested JSON file dynamically.\n","        :param nested_json: The JSON data (dict or list)\n","        :param parent_key: The key of the parent node (used for recursion)\n","        :param sep: Separator between parent and child keys\n","        :return: A flattened JSON\n","        \"\"\"\n","        items = []\n","        \n","\n","        if isinstance(nested_json, list):\n","            for i, item in enumerate(nested_json):\n","                items.extend(flatten_json(item, f'{parent_key}{sep}{i}', sep=sep).items())\n","        \n","    \n","        elif isinstance(nested_json, dict):\n","            for key, value in nested_json.items():\n","                new_key = f'{parent_key}{sep}{key}' if parent_key else key\n","                \n","\n","                if isinstance(value, (dict, list)):\n","                    items.extend(flatten_json(value, new_key, sep=sep).items())\n","                else:\n","                    items.append((new_key, value))\n","        \n","        return dict(items)"]},{"cell_type":"code","execution_count":null,"id":"f0f25c1f-cbdf-4610-868f-ef4fd7f26f45","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["from Workspace_Metadata import Workspace_Metadata\n","import json\n","#from flatten_json import flatten_json\n","\n","Wrks=Workspace_Metadata(headers_fabric)\n","ListID=Wrks.Workspace_List()\n","\n","\n","Wrks=Workspace_Metadata(headers_powerbi)\n","for workspaceid in ListID:\n","    print(workspaceid + 'workspace info')\n","    Wrks_Scan=Wrks.Workspace_Scan(workspaceid)\n","    response= Wrks.Workspace_Metadata(Wrks_Scan)\n","    \n","    json_output = json.dumps(response, indent=2)\n","    data = json.loads(json_output)  \n","    flattened_data = [flatten_json(workspace) for workspace in data['workspaces']]\n","\n","\n","    df = pd.DataFrame(flattened_data)\n","\n","    #spark_df = spark.createDataFrame(df)\n","    #spark_df.write.format(\"delta\").mode(\"append\").option(\"mergeSchema\", \"true\").saveAsTable(\"workspaceinfo\")\n"]}],"metadata":{"dependencies":{"environment":{"environmentId":"ca11e180-7f43-4d67-a1e3-59e0d42ef103","workspaceId":"2194de19-ac45-4d2c-82f1-b1eaf833f490"},"lakehouse":{"default_lakehouse":"d692902f-2159-42e8-9f5a-0b6981ce5b9a","default_lakehouse_name":"lh_fabric_ops_main1","default_lakehouse_workspace_id":"2194de19-ac45-4d2c-82f1-b1eaf833f490"}},"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"[email protected]"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5}

Hackaton/Workspace_Metadata.py

@@ -0,0 +1,66 @@
+
+import requests
+import json
+import pandas as pd
+
+
+
+class Workspace_Metadata:
+
+    def __init__(
+        self,
+        header: str,
+    ) -> None:
+        self.header= header
+
+    def Workspace_List(self)-> str:
+        """
+        List all wotkspace the token has access
+        : Token Header
+        :return: a list of workspace
+        """
+        url_get= "https://api.fabric.microsoft.com/v1/workspaces"
+        response_status = requests.get(url_get, headers=self.header)
+        response_data = response_status.json()
+        status = response_data.get('status')
+        workspace_ids = []
+
+        workspacevalue=response_data.get("value", {})
+        for workspaceid in workspacevalue:
+            workspace_id = workspaceid["id"]
+            if workspace_id:  # Check if 'id' exists before adding
+                workspace_ids.append(workspace_id)
+
+        return workspace_ids
+
+
+
+    def Workspace_Scan(self, workspacevalue:str)-> str:
+        """
+        Workspace Scan ID
+        :workspacevalue workspace id from Workspace_List
+        :return: scan id to be used in the workspace metadata function
+        """
+        url = "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo"
+
+        payload = {
+            "workspaces": [f"{workspacevalue}"],
+        }
+
+
+        response = requests.post(url,json=payload,  headers=self.header)
+        response.raise_for_status()
+        response = response.json()
+        scan_id= response.get("id", [])
+        return scan_id
+
+    def Workspace_Metadata(self,scan_id:str)-> str:
+        """
+        Workspace Metadata
+        :scan_id from the Workspace_Scan
+        :return: JSON response with the metadata information
+        """
+        url_get= f"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/{scan_id}"
+        response_status = requests.get(url_get, headers=self.header)
+        response_status = response_status.json()
+        return response_status

Hackaton/readme.txt

@@ -0,0 +1 @@
+