diff --git a/Hackaton/Helper_.py b/Hackaton/Helper_.py new file mode 100644 index 00000000..b8c492be --- /dev/null +++ b/Hackaton/Helper_.py @@ -0,0 +1,38 @@ +import requests +from msal import ConfidentialClientApplication + + +class Helper_Token: + def __init__( + self, + tenant_id: str, + app_id: str + ) -> None: + + self.tenant_id = tenant_id + self.app_id = app_id + + def get_token_ServPrincRead_token(self, scope:str, secret: str) -> str: + authority = f'https://login.microsoftonline.com/{self.tenant_id}' + app = ConfidentialClientApplication( + client_id = self.app_id, + authority=authority, + client_credential=secret + ) + + + token_response = app.acquire_token_for_client(scopes=[f'{scope}.default']) + #print(token_response) + access_token = token_response['access_token'] + + return access_token + + def get_token_ServPrincRead(access_token:str) -> str: + + headers = { + 'Authorization': f'Bearer {access_token}', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + return headers diff --git a/Hackaton/ScanAPIs_Execution.ipynb b/Hackaton/ScanAPIs_Execution.ipynb new file mode 100644 index 00000000..21c452bd --- /dev/null +++ b/Hackaton/ScanAPIs_Execution.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"markdown","id":"20e53853-65a0-40db-9b48-0f0600bc3ba0","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["#### Token Configuration\n"]},{"cell_type":"markdown","id":"9826e43d-c3c8-43e0-ac7a-4d893554c044","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["###### Variables"]},{"cell_type":"code","execution_count":null,"id":"afc18ee0-32d6-439d-a51a-77f2004292b9","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"outputs":[],"source":["# Replace these values with your actual authentication and resource details\n","tenant_id = \"\"\n","client_id = \"\"\n","client_secret = \"\" \n","\n","\n","\n"]},{"cell_type":"markdown","id":"122bb258-798f-49cd-bf0a-dd368eac90d9","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["### Getting Workspace metadata\n"]},{"cell_type":"markdown","id":"617454dd-f50e-4dd3-855b-f82e59055551","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["#### Fabric Token Scope"]},{"cell_type":"code","execution_count":2,"id":"a118999f-915e-4e75-8a48-94d841643755","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[{"data":{"application/vnd.livy.statement-meta+json":{"execution_finish_time":"2024-09-18T19:48:17.5807431Z","execution_start_time":"2024-09-18T19:48:16.7290055Z","livy_statement_state":"available","normalized_state":"finished","parent_msg_id":"bfe7afe5-9cab-43c7-8e95-1419564baf14","queued_time":"2024-09-18T19:43:12.4495132Z","session_id":"6822d424-8d13-4523-9b71-8ee604dd4808","session_start_time":null,"spark_pool":null,"state":"finished","statement_id":6,"statement_ids":[6]},"text/plain":["StatementMeta(, 6822d424-8d13-4523-9b71-8ee604dd4808, 6, Finished, Available, Finished)"]},"metadata":{},"output_type":"display_data"}],"source":["from Helper_ import Helper_Token\n","\n","resource = 'https://api.fabric.microsoft.com/'\n","\n","Helper_=Helper_Token(tenant_id=tenant_id,app_id=client_id)\n","Token=Helper_.get_token_ServPrincRead_token(scope=resource,secret=client_secret)\n","headers_fabric=Helper_Token.get_token_ServPrincRead(Token)\n"]},{"cell_type":"markdown","id":"eef44bdb-6c0c-40fb-a407-278cdb7ed7f5","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["##### POwer BI Token Scope"]},{"cell_type":"code","execution_count":null,"id":"a063aab6-b90d-4a2b-8dc4-386073b8a71c","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["from Helper_ import Helper_Token\n","\n","resource = 'https://analysis.windows.net/powerbi/api/'\n","\n","\n","Helper_=Helper_Token(tenant_id=tenant_id,app_id=client_id)\n","Token=Helper_.get_token_ServPrincRead_token(scope=resource,secret=client_secret)\n","headers_powerbi=Helper_Token.get_token_ServPrincRead(Token)\n"]},{"cell_type":"markdown","id":"684fb850-873c-4ed9-b38c-25cf20f58243","metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"source":["### Executing the Class to get the Metadata information\n","#### Fabric API is used to list workspaces\n","#### Power BI is used to list the workspaces metadata information"]},{"cell_type":"code","execution_count":null,"id":"1519bf73-27a1-42cb-bebc-d5a6f351af9c","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":[" def flatten_json(nested_json, parent_key='', sep='_'):\n"," \"\"\"\n"," Flatten a nested JSON file dynamically.\n"," :param nested_json: The JSON data (dict or list)\n"," :param parent_key: The key of the parent node (used for recursion)\n"," :param sep: Separator between parent and child keys\n"," :return: A flattened JSON\n"," \"\"\"\n"," items = []\n"," \n","\n"," if isinstance(nested_json, list):\n"," for i, item in enumerate(nested_json):\n"," items.extend(flatten_json(item, f'{parent_key}{sep}{i}', sep=sep).items())\n"," \n"," \n"," elif isinstance(nested_json, dict):\n"," for key, value in nested_json.items():\n"," new_key = f'{parent_key}{sep}{key}' if parent_key else key\n"," \n","\n"," if isinstance(value, (dict, list)):\n"," items.extend(flatten_json(value, new_key, sep=sep).items())\n"," else:\n"," items.append((new_key, value))\n"," \n"," return dict(items)"]},{"cell_type":"code","execution_count":null,"id":"f0f25c1f-cbdf-4610-868f-ef4fd7f26f45","metadata":{"jupyter":{"outputs_hidden":false,"source_hidden":false},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"outputs":[],"source":["from Workspace_Metadata import Workspace_Metadata\n","import json\n","#from flatten_json import flatten_json\n","\n","Wrks=Workspace_Metadata(headers_fabric)\n","ListID=Wrks.Workspace_List()\n","\n","\n","Wrks=Workspace_Metadata(headers_powerbi)\n","for workspaceid in ListID:\n"," print(workspaceid + 'workspace info')\n"," Wrks_Scan=Wrks.Workspace_Scan(workspaceid)\n"," response= Wrks.Workspace_Metadata(Wrks_Scan)\n"," \n"," json_output = json.dumps(response, indent=2)\n"," data = json.loads(json_output) \n"," flattened_data = [flatten_json(workspace) for workspace in data['workspaces']]\n","\n","\n"," df = pd.DataFrame(flattened_data)\n","\n"," #spark_df = spark.createDataFrame(df)\n"," #spark_df.write.format(\"delta\").mode(\"append\").option(\"mergeSchema\", \"true\").saveAsTable(\"workspaceinfo\")\n"]}],"metadata":{"dependencies":{"environment":{"environmentId":"ca11e180-7f43-4d67-a1e3-59e0d42ef103","workspaceId":"2194de19-ac45-4d2c-82f1-b1eaf833f490"},"lakehouse":{"default_lakehouse":"d692902f-2159-42e8-9f5a-0b6981ce5b9a","default_lakehouse_name":"lh_fabric_ops_main1","default_lakehouse_workspace_id":"2194de19-ac45-4d2c-82f1-b1eaf833f490"}},"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"display_name":"Synapse PySpark","language":"Python","name":"synapse_pyspark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"nteract":{"version":"nteract-front-end@1.0.0"},"spark_compute":{"compute_id":"/trident/default"},"synapse_widget":{"state":{},"version":"0.1"},"widgets":{}},"nbformat":4,"nbformat_minor":5} diff --git a/Hackaton/Workspace_Metadata.py b/Hackaton/Workspace_Metadata.py new file mode 100644 index 00000000..6f1cecf1 --- /dev/null +++ b/Hackaton/Workspace_Metadata.py @@ -0,0 +1,66 @@ + +import requests +import json +import pandas as pd + + + +class Workspace_Metadata: + + def __init__( + self, + header: str, + ) -> None: + self.header= header + + def Workspace_List(self)-> str: + """ + List all wotkspace the token has access + : Token Header + :return: a list of workspace + """ + url_get= "https://api.fabric.microsoft.com/v1/workspaces" + response_status = requests.get(url_get, headers=self.header) + response_data = response_status.json() + status = response_data.get('status') + workspace_ids = [] + + workspacevalue=response_data.get("value", {}) + for workspaceid in workspacevalue: + workspace_id = workspaceid["id"] + if workspace_id: # Check if 'id' exists before adding + workspace_ids.append(workspace_id) + + return workspace_ids + + + + def Workspace_Scan(self, workspacevalue:str)-> str: + """ + Workspace Scan ID + :workspacevalue workspace id from Workspace_List + :return: scan id to be used in the workspace metadata function + """ + url = "https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo" + + payload = { + "workspaces": [f"{workspacevalue}"], + } + + + response = requests.post(url,json=payload, headers=self.header) + response.raise_for_status() + response = response.json() + scan_id= response.get("id", []) + return scan_id + + def Workspace_Metadata(self,scan_id:str)-> str: + """ + Workspace Metadata + :scan_id from the Workspace_Scan + :return: JSON response with the metadata information + """ + url_get= f"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/{scan_id}" + response_status = requests.get(url_get, headers=self.header) + response_status = response_status.json() + return response_status diff --git a/Hackaton/readme.txt b/Hackaton/readme.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/Hackaton/readme.txt @@ -0,0 +1 @@ +