Add dashboards to migration progress dashboard (#3314)

## Changes
Add linting resources to migration progress dashboard

### Linked issues

- [x] Depends on #3424
- [x] Progresses #3045 
- [x] Breaks up #3112

### Functionality

- [x] modified existing dashboard: `Migration [main]`

### Tests

- [x] modified unit tests
- [x] modified integration tests

---------

Co-authored-by: Guenia Izquierdo Delgado <[email protected]>

src/databricks/labs/ucx/queries/progress/main/01_0_percentage_migration_progress.sql → src/databricks/labs/ucx/queries/progress/main/01_00_percentage_migration_progress.sql

@@ -2,4 +2,4 @@
 SELECT
     ROUND(100 * try_divide(COUNT_IF(SIZE(failures) = 0), COUNT(*)), 2) AS percentage
 FROM ucx_catalog.multiworkspace.objects_snapshot
-WHERE object_type IN ('ClusterInfo', 'Grant', 'JobInfo', 'PipelineInfo', 'PolicyInfo', 'Table', 'Udf')
+WHERE object_type IN ('ClusterInfo', 'Grant', 'Dashboard', 'JobInfo', 'PipelineInfo', 'PolicyInfo', 'Table', 'Udf')

src/databricks/labs/ucx/queries/progress/main/01_5_percentage_table_migration_progress.sql → src/databricks/labs/ucx/queries/progress/main/01_05_percentage_table_migration_progress.sql

@@ -1,4 +1,4 @@
-/* --title 'Table migration progress (%)' --width 2 */
+/* --title 'Table migration progress (%)' */
 SELECT
     ROUND(100 * TRY_DIVIDE(COUNT_IF(SIZE(failures) = 0), COUNT(*)), 2) AS percentage
 FROM ucx_catalog.multiworkspace.objects_snapshot

src/databricks/labs/ucx/queries/progress/main/01_08_percentage_dashboard_migration_progress.sql

@@ -0,0 +1,5 @@
+/* --title 'Dashboard progress (%)' */
+SELECT
+    ROUND(100 * TRY_DIVIDE(COUNT_IF(SIZE(failures) = 0), COUNT(*)), 2) AS percentage
+FROM ucx_catalog.multiworkspace.objects_snapshot
+WHERE object_type = "Dashboard"

src/databricks/labs/ucx/queries/progress/main/02_4_migration_status_by_owner_overview.sql

@@ -1,6 +1,6 @@
 /* --title 'Overview' --description 'Tables and views migration' --width 5 */
 WITH migration_statuses AS (
-    SELECT *
+    SELECT owner, failures
     FROM ucx_catalog.multiworkspace.objects_snapshot
     WHERE object_type = 'Table'
 )

src/databricks/labs/ucx/queries/progress/main/03_00_dashboards.md

@@ -0,0 +1,8 @@
+# Dashboards
+
+This section shows Unity Catalog compatability issues found while linting dashboards. There are two kinds of changes to
+perform:
+- Data asset reference, i.e. references to Hive metastore tables and views or direct filesystem access (dfsa), these
+  references should be updated to refer to their Unity Catalog counterparts.
+- Linting compatability issues, e.g. using RDDs or directly accessing the Spark context, these issues should be resolved
+  by following the instructions stated with the issue.

src/databricks/labs/ucx/queries/progress/main/03_01_dashboards_pending_migration.sql

@@ -0,0 +1,4 @@
+/* --title 'Dashboards pending migration' --height 6 */
+SELECT COUNT(*) AS count
+FROM ucx_catalog.multiworkspace.objects_snapshot
+WHERE object_type = 'Dashboard' AND SIZE(failures) > 0

src/databricks/labs/ucx/queries/progress/main/03_02_dashboards_pending_migration_by_owner_bar_graph.sql

@@ -0,0 +1,23 @@
+/*
+--title 'Dashboards pending migration'
+--width 5
+--overrides '{"spec": {
+    "version": 3,
+    "widgetType": "bar",
+    "encodings": {
+        "x": {"fieldName": "owner", "scale": {"type": "categorical"}, "displayName": "owner"},
+        "y": {"fieldName": "count", "scale": {"type": "quantitative"}, "displayName": "count"}
+    }
+}}'
+*/
+WITH owners_with_failures AS (
+    SELECT owner
+    FROM ucx_catalog.multiworkspace.objects_snapshot
+    WHERE object_type = 'Dashboard' AND SIZE(failures) > 0
+)
+
+SELECT
+    owner,
+    COUNT(1) AS count
+FROM owners_with_failures
+GROUP BY owner

src/databricks/labs/ucx/queries/progress/main/03_03_dashboards_migrated.sql

@@ -0,0 +1,4 @@
+/* --title 'Dashboards migrated' --height 6 */
+SELECT COUNT(*) AS count
+FROM ucx_catalog.multiworkspace.objects_snapshot
+WHERE object_type = 'Dashboard' AND SIZE(failures) == 0

src/databricks/labs/ucx/queries/progress/main/03_04_dashboards_pending_migration_by_owner_overview.sql

@@ -0,0 +1,15 @@
+/* --title 'Dashboard pending migration' --width 5 */
+WITH migration_statuses AS (
+    SELECT owner, failures
+    FROM ucx_catalog.multiworkspace.objects_snapshot
+    WHERE object_type = 'Dashboard'
+)
+
+SELECT
+    owner,
+    DOUBLE(CEIL(100 * COUNT_IF(SIZE(failures) = 0) / SUM(COUNT(*)) OVER (PARTITION BY owner), 2)) AS percentage,
+    COUNT(*) AS total,
+    COUNT_IF(SIZE(failures) = 0) AS total_migrated,
+    COUNT_IF(SIZE(failures) > 0) AS total_not_migrated
+FROM migration_statuses
+GROUP BY owner

src/databricks/labs/ucx/queries/progress/main/03_05_dashboards_pending_migration.sql

@@ -0,0 +1,35 @@
+/*
+--title 'Dashboards pending migration'
+--width 6
+--overrides '{"spec":{
+    "encodings":{
+      "columns": [
+        {"fieldName": "workspace_id", "booleanValues": ["false", "true"], "type": "string", "displayAs": "string", "title": "workspace_id"},
+        {"fieldName": "owner", "booleanValues": ["false", "true"], "type": "string", "displayAs": "string", "title": "owner"},
+        {"fieldName": "name", "title": "Name", "type": "string", "displayAs": "link", "linkUrlTemplate": "{{ dashboard_link }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "booleanValues": ["false", "true"]},
+        {"fieldName": "dashboard_type", "booleanValues": ["false", "true"], "type": "string", "displayAs": "string", "title": "dashboard_type"},
+        {"fieldName": "failure", "booleanValues": ["false", "true"], "type": "string", "displayAs": "string", "title": "failure"}
+      ]},
+    "invisibleColumns": [
+      {"fieldName": "dashboard_link", "title": "dashboard_link", "type": "string", "displayAs": "string", "booleanValues": ["false", "true"]}
+    ]
+  }}'
+*/
+SELECT
+  workspace_id,
+  owner,
+  data.name AS name,
+  CASE
+      -- Simple heuristic to differentiate between Redash and Lakeview dashboards
+      WHEN CONTAINS(data.id, '-') THEN 'Redash'
+      ELSE 'Lakeview'
+  END AS dashboard_type,
+  EXPLODE(failures) AS failure,
+  -- Below are invisible column(s) used in links url templates
+  CASE
+    WHEN CONTAINS(data.id, '-') THEN CONCAT('/sql/dashboards/', data.id)
+    ELSE CONCAT('/dashboardsv3/', data.id, '/published')
+  END AS dashboard_link
+FROM ucx_catalog.multiworkspace.objects_snapshot
+WHERE object_type = 'Dashboard' AND SIZE(failures) > 0
+ORDER BY workspace_id, owner, name, failure

tests/integration/progress/test_workflows.py

@@ -11,6 +11,7 @@
 @retried(on=[NotFound, InvalidParameterValue], timeout=dt.timedelta(minutes=12))
 def test_running_real_migration_progress_job(installation_ctx: MockInstallationContext) -> None:
     """Ensure that the migration-progress workflow can complete successfully."""
+
     # Limit the resources crawled by the assessment
     source_schema = installation_ctx.make_schema()
     installation_ctx.make_table(schema_name=source_schema.name)