diff --git a/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json b/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json new file mode 100644 index 00000000..7d6e0158 --- /dev/null +++ b/templates/Unlock SharePoint Oversharing Insights/Unlock SharePoint Oversharing Insights.json @@ -0,0 +1,1798 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "LS_Lakehouse-SQLEndPoint": { + "type": "string" + }, + "LS_Office365_MGDC": { + "type": "string" + } + }, + "variables": {}, + "resources": [ + { + "name": "Unlock SharePoint Oversharing Insights", + "type": "pipelines", + "apiVersion": "2018-06-01", + "properties": { + "description": "This Pipeline will extract the data from Microsoft Graph Data connect to Lakehouse for getting oversharing insights in your organization. This pipeline needs some dependencies and steps documented here. https://go.microsoft.com/fwlink/?linkid=2300611", + "activities": [ + { + "name": "Merge Sites and Permissions To Final Table", + "type": "TridentNotebook", + "dependsOn": [ + { + "activity": "Extract Sites From MGDC", + "dependencyConditions": [ + "Completed" + ] + }, + { + "activity": "Extract Permissions From MGDC", + "dependencyConditions": [ + "Completed" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "044e9c94-b555-42eb-af0d-b9b316b0c501", + "workspaceId": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "parameters": { + "sitesStagingTableName": { + "value": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName", + "type": "Expression" + }, + "type": "string" + }, + "sitesFinalTableName": { + "value": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesFinalTableName", + "type": "Expression" + }, + "type": "string" + }, + "permissionsStagingTableName": { + "value": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsStagingTableName", + "type": "Expression" + }, + "type": "string" + }, + "permissionsFinalTableName": { + "value": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsFinalTableName", + "type": "Expression" + }, + "type": "string" + }, + "runId": { + "value": { + "value": "@pipeline().RunId", + "type": "Expression" + }, + "type": "string" + }, + "lakehouseName": { + "value": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + }, + "type": "string" + }, + "workspaceId": { + "value": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "type": "string" + } + } + } + }, + { + "name": "Set End Snapshot Date", + "description": "", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "EndTime", + "value": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).EndSnapshotDate\n\n", + "type": "Expression" + } + } + }, + { + "name": "Read Last Snapshot Dates", + "type": "TridentNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "31fc030f-0e0c-46bc-ae89-281bd5c9272a", + "workspaceId": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "parameters": { + "sitesStagingTableName": { + "value": { + "value": "@concat(pipeline().parameters.SitesTableName,pipeline().parameters.StagingSuffix)", + "type": "Expression" + }, + "type": "string" + }, + "sitesFinalTableName": { + "value": { + "value": "@pipeline().parameters.SitesTableName", + "type": "Expression" + }, + "type": "string" + }, + "permissionsStagingTableName": { + "value": { + "value": "@concat(pipeline().parameters.PermissionsTableName,pipeline().parameters.StagingSuffix)", + "type": "Expression" + }, + "type": "string" + }, + "permissionsFinalTableName": { + "value": { + "value": "@pipeline().parameters.PermissionsTableName", + "type": "Expression" + }, + "type": "string" + }, + "runId": { + "value": { + "value": "@pipeline().RunId", + "type": "Expression" + }, + "type": "string" + }, + "lakehouseName": { + "value": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + }, + "type": "string" + }, + "workspaceId": { + "value": { + "value": "@pipeline().DataFactory", + "type": "Expression" + }, + "type": "string" + }, + "endTime": { + "value": { + "value": "@coalesce(pipeline().parameters.EndTime,formatDateTime(adddays(startOfDay(pipeline().TriggerTime),-2),'yyyy-MM-ddTHH:mm:ssZ'))\n", + "type": "Expression" + }, + "type": "string" + } + } + } + }, + { + "name": "Set Sites Snapshot Date", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "StartTimeForSites", + "value": { + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesSnapshotDate)", + "type": "Expression" + } + } + }, + { + "name": "Set Permissions Snapshot Date", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "StartTimeForPermissions", + "value": { + "value": "@coalesce(pipeline().parameters.StartTime, json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsSnapshotDate)", + "type": "Expression" + } + } + }, + { + "name": "Sites View - Lakehouse SQLEndpoint", + "type": "Script", + "dependsOn": [ + { + "activity": "Merge Sites and Permissions To Final Table", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "scripts": [ + { + "type": "Query", + "text": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesView", + "type": "Expression" + } + } + ], + "scriptBlockExecutionTimeout": "02:00:00", + "database": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + } + }, + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" + } + }, + { + "name": "Permissions View - Lakehouse SQLEndpoint", + "type": "Script", + "dependsOn": [ + { + "activity": "Merge Sites and Permissions To Final Table", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "scripts": [ + { + "type": "Query", + "text": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsView", + "type": "Expression" + } + } + ], + "scriptBlockExecutionTimeout": "02:00:00", + "database": { + "value": "@pipeline().parameters.LakehouseName", + "type": "Expression" + } + }, + "externalReferences": { + "connection": "[parameters('LS_Lakehouse-SQLEndPoint')]" + } + }, + { + "name": "Does Permissions Data Exists", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "PermissionsDataExists", + "value": { + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsDataExists)", + "type": "Expression" + } + } + }, + { + "name": "Does Sites Data Exists", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Read Last Snapshot Dates", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "SitesDataExists", + "value": { + "value": "@bool(json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesDataExists)", + "type": "Expression" + } + } + }, + { + "name": "Extract Sites From MGDC", + "description": "", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Does Sites Data Exists", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set Sites Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set End Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "expression": { + "value": "@and(and(variables('SitesDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForSites')) , startOfDay(variables('EndTime')) ))\n\n ", + "type": "Expression" + }, + "ifFalseActivities": [ + { + "name": "Extract Sites To Staging", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "Office365Source", + "dateFilterColumn": "SnapshotDate", + "startTime": { + "value": "@variables('StartTimeForSites')", + "type": "Expression" + }, + "endTime": { + "value": "@variables('EndTime')", + "type": "Expression" + }, + "datasetSettings": { + "annotations": [], + "type": "Office365Table", + "schema": [], + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointSites_v1" + }, + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + } + }, + "DataFilter": { + "value": "@pipeline().parameters.SitesFilter", + "type": "Expression" + } + }, + "sink": { + "type": "LakehouseTableSink", + "tableActionOption": "OverwriteSchema", + "partitionOption": "None", + "datasetSettings": { + "annotations": [], + "linkedService": { + "name": "e629b492_d56f_43cc_a93f_3558c83b2f5d", + "properties": { + "annotations": [], + "type": "Lakehouse", + "typeProperties": { + "workspaceId": "@pipeline().DataFactory", + "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", + "rootFolder": "Tables" + } + } + }, + "type": "LakehouseTable", + "schema": [], + "typeProperties": { + "schema": "dbo", + "table": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).SitesStagingTableName", + "type": "Expression" + } + } + } + }, + "enableStaging": true, + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "name": "ptenant", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ptenant", + "physicalType": "string" + } + }, + { + "source": { + "name": "Id", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Id", + "physicalType": "string" + } + }, + { + "source": { + "name": "Url", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Url", + "physicalType": "string" + } + }, + { + "source": { + "name": "ArchiveState", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ArchiveState", + "physicalType": "string" + } + }, + { + "source": { + "name": "RootWeb_Configuration", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "RootWeb_Configuration", + "physicalType": "long" + } + }, + { + "source": { + "name": "RootWeb_Id", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RootWeb_Id", + "physicalType": "string" + } + }, + { + "source": { + "name": "RootWeb_Title", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RootWeb_Title", + "physicalType": "string" + } + }, + { + "source": { + "name": "RootWeb_WebTemplate", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RootWeb_WebTemplate", + "physicalType": "string" + } + }, + { + "source": { + "name": "RootWeb_WebTemplateId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RootWeb_WebTemplateId", + "physicalType": "string" + } + }, + { + "source": { + "name": "RootWeb_LastItemModifiedDate", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "RootWeb_LastItemModifiedDate", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "WebCount", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "WebCount", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageQuota", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageQuota", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageUsed", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageUsed", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageMetrics_MetadataSize", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageMetrics_MetadataSize", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageMetrics_TotalFileCount", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageMetrics_TotalFileCount", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageMetrics_TotalFileStreamSize", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageMetrics_TotalFileStreamSize", + "physicalType": "long" + } + }, + { + "source": { + "name": "StorageMetrics_TotalSize", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "StorageMetrics_TotalSize", + "physicalType": "long" + } + }, + { + "source": { + "name": "GroupId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "GroupId", + "physicalType": "string" + } + }, + { + "source": { + "name": "GeoLocation", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "GeoLocation", + "physicalType": "string" + } + }, + { + "source": { + "name": "IsInRecycleBin", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsInRecycleBin", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "RecycleBinItemCount", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "RecycleBinItemCount", + "physicalType": "long" + } + }, + { + "source": { + "name": "RecycleBinItemSize", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "RecycleBinItemSize", + "physicalType": "long" + } + }, + { + "source": { + "name": "SecondStageRecycleBinStorageUsage", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "SecondStageRecycleBinStorageUsage", + "physicalType": "long" + } + }, + { + "source": { + "name": "IsTeamsConnectedSite", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsTeamsConnectedSite", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "IsTeamsChannelSite", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsTeamsChannelSite", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "TeamsChannelType", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "TeamsChannelType", + "physicalType": "string" + } + }, + { + "source": { + "name": "IsCommunicationSite", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsCommunicationSite", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "IsHubSite", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsHubSite", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "HubSiteId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "HubSiteId", + "physicalType": "string" + } + }, + { + "source": { + "name": "IsOneDrive", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsOneDrive", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "BlockAccessFromUnmanagedDevices", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "BlockAccessFromUnmanagedDevices", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "BlockDownloadOfAllFilesOnUnmanagedDevices", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "BlockDownloadOfViewableFilesOnUnmanagedDevices", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "IsExternalSharingEnabled", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "IsExternalSharingEnabled", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "Privacy", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Privacy", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareByEmailEnabled", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "ShareByEmailEnabled", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "ShareByLinkEnabled", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "ShareByLinkEnabled", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "SiteConnectedToPrivateGroup", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "SiteConnectedToPrivateGroup", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "SensitivityLabelInfo_DisplayName", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SensitivityLabelInfo_DisplayName", + "physicalType": "string" + } + }, + { + "source": { + "name": "SensitivityLabelInfo_Id", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SensitivityLabelInfo_Id", + "physicalType": "string" + } + }, + { + "source": { + "name": "Classification", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Classification", + "physicalType": "string" + } + }, + { + "source": { + "name": "IBMode", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "IBMode", + "physicalType": "string" + } + }, + { + "source": { + "name": "IBSegments", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "IBSegments", + "physicalType": "string" + } + }, + { + "source": { + "name": "RelatedGroupId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RelatedGroupId", + "physicalType": "string" + } + }, + { + "source": { + "name": "Owner_AadObjectId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Owner_AadObjectId", + "physicalType": "string" + } + }, + { + "source": { + "name": "Owner_Email", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Owner_Email", + "physicalType": "string" + } + }, + { + "source": { + "name": "Owner_Name", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Owner_Name", + "physicalType": "string" + } + }, + { + "source": { + "name": "Owner_UPN", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Owner_UPN", + "physicalType": "string" + } + }, + { + "source": { + "name": "SecondaryContact_AadObjectId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SecondaryContact_AadObjectId", + "physicalType": "string" + } + }, + { + "source": { + "name": "SecondaryContact_Email", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SecondaryContact_Email", + "physicalType": "string" + } + }, + { + "source": { + "name": "SecondaryContact_Name", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SecondaryContact_Name", + "physicalType": "string" + } + }, + { + "source": { + "name": "SecondaryContact_UPN", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SecondaryContact_UPN", + "physicalType": "string" + } + }, + { + "source": { + "name": "ReadLocked", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "ReadLocked", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "ReadOnly", + "type": "Boolean", + "physicalType": "boolean" + }, + "sink": { + "name": "ReadOnly", + "physicalType": "boolean" + } + }, + { + "source": { + "name": "CreatedTime", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "CreatedTime", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "LastSecurityModifiedDate", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "LastSecurityModifiedDate", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "Operation", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Operation", + "physicalType": "string" + } + }, + { + "source": { + "name": "LastUserAccessDate", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "LastUserAccessDate", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "SnapshotDate", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "SnapshotDate", + "physicalType": "timestamp" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + }, + "columnFlattenSettings": { + "treatArrayAsString": false, + "treatStructAsString": false, + "flattenColumnDelimiter": "_" + } + } + } + } + ], + "ifTrueActivities": [ + { + "name": "Do Not Extract Sites", + "description": "Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "type": "Fail", + "dependsOn": [], + "typeProperties": { + "message": "Error: -999 - Do Not Extract Sites - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "errorCode": "-999" + } + } + ] + } + }, + { + "name": "Extract Permissions From MGDC", + "description": "", + "type": "IfCondition", + "dependsOn": [ + { + "activity": "Set Permissions Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Does Permissions Data Exists", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Set End Snapshot Date", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "typeProperties": { + "expression": { + "value": "@and(and(variables('PermissionsDataExists'),\n not(pipeline().parameters.ForceFullWhenDataExists)\t \n\t ), greaterOrEquals( startOfDay(variables('StartTimeForPermissions')) , startOfDay(variables('EndTime')) ))", + "type": "Expression" + }, + "ifFalseActivities": [ + { + "name": "Extract Permissions To Staging", + "type": "Copy", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "source": { + "type": "Office365Source", + "dateFilterColumn": "SnapshotDate", + "startTime": { + "value": "@variables('StartTimeForPermissions')", + "type": "Expression" + }, + "endTime": { + "value": "@variables('EndTime')", + "type": "Expression" + }, + "datasetSettings": { + "annotations": [], + "type": "Office365Table", + "schema": [], + "typeProperties": { + "tableName": "BasicDataSet_v0.SharePointPermissions_v1" + }, + "DataFilter": { + "value": "@pipeline().parameters.PermissionsFilter", + "type": "Expression" + }, + "externalReferences": { + "connection": "[parameters('LS_Office365_MGDC')]" + } + } + }, + "sink": { + "type": "LakehouseTableSink", + "tableActionOption": "OverwriteSchema", + "partitionOption": "None", + "datasetSettings": { + "annotations": [], + "linkedService": { + "name": "f589a5e3_9e9f_4bbe_8e32_55faadf2c82f", + "properties": { + "annotations": [], + "type": "Lakehouse", + "typeProperties": { + "workspaceId": "@pipeline().DataFactory", + "artifactId": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).LakehouseId", + "rootFolder": "Tables" + } + } + }, + "type": "LakehouseTable", + "schema": [], + "typeProperties": { + "schema": "dbo", + "table": { + "value": "@json(activity('Read Last Snapshot Dates').output.result.exitValue).PermissionsStagingTableName", + "type": "Expression" + } + } + } + }, + "enableStaging": true, + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { + "name": "ptenant", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ptenant", + "physicalType": "string" + } + }, + { + "source": { + "name": "SiteId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SiteId", + "physicalType": "string" + } + }, + { + "source": { + "name": "WebId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "WebId", + "physicalType": "string" + } + }, + { + "source": { + "name": "ListId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ListId", + "physicalType": "string" + } + }, + { + "source": { + "name": "ItemType", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ItemType", + "physicalType": "string" + } + }, + { + "source": { + "name": "ItemURL", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ItemURL", + "physicalType": "string" + } + }, + { + "source": { + "name": "FileExtension", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "FileExtension", + "physicalType": "string" + } + }, + { + "source": { + "name": "RoleDefinition", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "RoleDefinition", + "physicalType": "string" + } + }, + { + "source": { + "name": "LinkId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "LinkId", + "physicalType": "string" + } + }, + { + "source": { + "name": "ScopeId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ScopeId", + "physicalType": "string" + } + }, + { + "source": { + "name": "LinkScope", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "LinkScope", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_Type", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_Type", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_Name", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_Name", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_Email", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_Email", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_TypeV2", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_TypeV2", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_UPN", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_UPN", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_AADObjectId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "SharedWith_AADObjectId", + "physicalType": "string" + } + }, + { + "source": { + "name": "SharedWith_UserCount", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "SharedWith_UserCount", + "physicalType": "long" + } + }, + { + "source": { + "name": "Operation", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "Operation", + "physicalType": "string" + } + }, + { + "source": { + "name": "SnapshotDate", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "SnapshotDate", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "ShareCreatedBy_Type", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareCreatedBy_Type", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareCreatedBy_Name", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareCreatedBy_Name", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareCreatedBy_Email", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareCreatedBy_Email", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareCreatedBy_UPN", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareCreatedBy_UPN", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareCreatedTime", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "ShareCreatedTime", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "ShareLastModifiedBy_Type", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareLastModifiedBy_Type", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareLastModifiedBy_Name", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareLastModifiedBy_Name", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareLastModifiedBy_Email", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareLastModifiedBy_Email", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareLastModifiedBy_UPN", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "ShareLastModifiedBy_UPN", + "physicalType": "string" + } + }, + { + "source": { + "name": "ShareLastModifiedTime", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "ShareLastModifiedTime", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "ShareExpirationTime", + "type": "DateTime", + "physicalType": "datetime" + }, + "sink": { + "name": "ShareExpirationTime", + "physicalType": "timestamp" + } + }, + { + "source": { + "name": "TotalUserCount", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "TotalUserCount", + "physicalType": "long" + } + }, + { + "source": { + "name": "ListItemId", + "type": "Int64", + "physicalType": "int64" + }, + "sink": { + "name": "ListItemId", + "physicalType": "long" + } + }, + { + "source": { + "name": "UniqueId", + "type": "String", + "physicalType": "string" + }, + "sink": { + "name": "UniqueId", + "physicalType": "string" + } + } + ], + "typeConversion": true, + "typeConversionSettings": { + "allowDataTruncation": true, + "treatBooleanAsNumber": false + }, + "columnFlattenSettings": { + "treatArrayAsString": false, + "treatStructAsString": false, + "flattenColumnDelimiter": "_" + } + } + } + } + ], + "ifTrueActivities": [ + { + "name": "Do Not Extract Permissions", + "description": "Do Not Extract Permissions - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "type": "Fail", + "dependsOn": [], + "typeProperties": { + "message": "Error: -999 - Do Not Extract Permissions - Data already exists and (both start time and end time same or start time is greater than end time) and ForceFullWhenDataExists parameter set to false", + "errorCode": "-999" + } + } + ] + } + }, + { + "name": "Permission Validations Failed", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Extract Permissions From MGDC", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "PermissionsExtractValidationsFailed", + "value": "true" + } + }, + { + "name": "Site Validations Failed", + "description": "true", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Extract Sites From MGDC", + "dependencyConditions": [ + "Failed" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "PermissionsExtractValidationsFailed", + "value": "true" + } + }, + { + "name": "Pipeline Success", + "description": "If this activity completed then Pipeline Completed Successfully", + "type": "SetVariable", + "dependsOn": [ + { + "activity": "Sites View - Lakehouse SQLEndpoint", + "dependencyConditions": [ + "Succeeded" + ] + }, + { + "activity": "Permissions View - Lakehouse SQLEndpoint", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "variableName": "PipelineFinalStatus", + "value": "Pipeline Completed Successfully" + } + } + ], + "parameters": { + "PermissionsTableName": { + "type": "string", + "defaultValue": "Permissions" + }, + "SitesTableName": { + "type": "string", + "defaultValue": "Sites" + }, + "StagingSuffix": { + "type": "string", + "defaultValue": "_Staging" + }, + "LakehouseName": { + "type": "string", + "defaultValue": "<>" + }, + "StartTime": { + "type": "string" + }, + "EndTime": { + "type": "string" + }, + "SitesFilter": { + "type": "string" + }, + "PermissionsFilter": { + "type": "string" + }, + "ForceFullWhenDataExists": { + "type": "bool", + "defaultValue": false + } + }, + "variables": { + "StartTimeForSites": { + "type": "String" + }, + "EndTime": { + "type": "String" + }, + "StartTimeForPermissions": { + "type": "String" + }, + "SitesDataExists": { + "type": "Boolean", + "defaultValue": false + }, + "PermissionsDataExists": { + "type": "Boolean", + "defaultValue": false + }, + "SiteExtractValidationsFailed": { + "type": "String", + "defaultValue": "false" + }, + "PermissionsExtractValidationsFailed": { + "type": "String", + "defaultValue": "false" + }, + "PipelineFinalStatus": { + "type": "String" + } + }, + "lastModifiedByObjectId": "1a2fbd2b-16c9-4b04-96de-be6e856e5333", + "lastPublishTime": "2025-01-03T20:19:45Z", + "extraTridentInfo": "{\"folderObjectId\":\"0cc8ced7-35a7-4dcb-94af-3b4f54c381b0\",\"capacityObjectId\":\"633251C5-A944-4D3E-B523-E61846F67005\",\"artifactType\":\"Pipeline\",\"provisionState\":\"Active\",\"lastUpdatedDate\":\"2025-01-03T20:19:43.984Z\",\"createdDate\":\"2024-11-11T21:15:17.805Z\",\"ownerUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"ownerUserId\":619869,\"createdByUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"createdByUserId\":619869,\"modifiedByUser\":{\"id\":619869,\"name\":\"Admin Archimedes\",\"objectId\":\"1a2fbd2b-16c9-4b04-96de-be6e856e5333\",\"userPrincipalName\":\"admin@odspimax.onmicrosoft.com\",\"aadAppId\":null,\"type\":null},\"modifiedByUserId\":619869,\"artifactRelations\":[{\"dependentArtifactObjectId\":\"044e9c94-b555-42eb-af0d-b9b316b0c501\",\"settings\":0},{\"dependentArtifactObjectId\":\"31fc030f-0e0c-46bc-ae89-281bd5c9272a\",\"settings\":0}],\"permissions\":71,\"artifactPermissions\":7,\"systemArtifactType\":\"None\",\"etag\":\"\\\"602479d8aa384b309be9371e34286d2a\\\"\"}" + }, + "dependsOn": [] + } + ] +} \ No newline at end of file diff --git a/templates/Unlock SharePoint Oversharing Insights/manifest.json b/templates/Unlock SharePoint Oversharing Insights/manifest.json new file mode 100644 index 00000000..1e31861e --- /dev/null +++ b/templates/Unlock SharePoint Oversharing Insights/manifest.json @@ -0,0 +1,31 @@ +{ + "name": "Unlock SharePoint Oversharing Insights", + "description": "This Pipeline will extract the data from Microsoft Graph Data connect to Lakehouse for getting oversharing insights in your organization. This pipeline needs some dependencies and steps documented here. https://go.microsoft.com/fwlink/?linkid=2300611", + "image": "NotebookMerge Sites andPermissions To...Set variableSet End SnapshotDateNotebookRead Last SnapshotDatesSet variableSet Sites SnapshotDateSet variableSet PermissionsSnapshot DateScriptSites View -Lakehouse...ScriptPermissions View -Lakehouse...Set variableDoes PermissionsData ExistsSet variableDoes Sites DataExistsIf ConditionExtract Sites FromMGDCTrueDo NotExtract...+FalseExtractSites To...+If ConditionExtract PermissionsFrom MGDCTrueDo NotExtract...+FalseExtractPermissio...+Set variablePermissionValidations FailedSet variableSite ValidationsFailedSet variablePipeline Success", + "icons": [ + "TridentNotebook", + "SetVariable", + "TridentNotebook" + ], + "requires": { + "linkedservices": { + "LS_Lakehouse-SQLEndPoint": { + "supportTypes": [ + "AzureSqlDatabase" + ] + }, + "LS_Office365_MGDC": { + "supportTypes": [ + "Office365" + ] + } + } + }, + "author": "Microsoft", + "annotations": [], + "services": [], + "categories": [], + "scope": [ + "PBI" + ] +} \ No newline at end of file