feat(database-clusters): Implement V1 of the database clusters tab in…

… Snuba Admin (#6345)

Implemented V1 of the database clusters tab in Snuba Admin. The purpose
of this tab is to show the nodes on which clickhouse is running as well
as their versions.

Future work: Instead of using a raw table, there should be groupings and
relationships that allow us to better visualize the cluster setup (e.g.
clusters should be grouped together). We should also add more metadata
so that it's easier to understand the topology of our clusters
(distributed nodes should be differentiated from regular storage nodes).

<img width="1462" alt="image"
src="https://github.com/user-attachments/assets/5b9d2331-1c37-4ac7-8da5-8b188215f02d">

.vscode/settings.json

@@ -40,7 +40,7 @@
 
   "[typescriptreact]": {
     "editor.formatOnSave": true,
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "vscode.typescript-language-features"
   },
 
   "[less]": {

snuba/admin/clickhouse/database_clusters.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+from typing import Sequence
+
+from snuba.admin.clickhouse.common import get_ro_node_connection
+from snuba.admin.clickhouse.nodes import get_storage_info
+from snuba.clusters.cluster import ClickhouseClientSettings
+
+
+@dataclass(frozen=True)
+class Node:
+    cluster: str
+    host_name: str
+    host_address: str
+    shard: int
+    replica: int
+    version: str
+
+
+@dataclass(frozen=True)
+class HostInfo:
+    host: str
+    port: int
+    storage_name: str
+
+
+def get_node_info() -> Sequence[Node]:
+    node_info = []
+    hosts = set()
+    for storage_info in get_storage_info():
+        for node in storage_info["dist_nodes"]:
+            hosts.add(
+                HostInfo(node["host"], node["port"], storage_info["storage_name"])
+            )
+
+        for node in storage_info["local_nodes"]:
+            hosts.add(
+                HostInfo(node["host"], node["port"], storage_info["storage_name"])
+            )
+
+    for host_info in hosts:
+        connection = get_ro_node_connection(
+            host_info.host,
+            host_info.port,
+            host_info.storage_name,
+            ClickhouseClientSettings.QUERY,
+        )
+        nodes = [
+            Node(*result)
+            for result in connection.execute(
+                "SELECT cluster, host_name, host_address, shard_num, replica_num, version() FROM system.clusters WHERE is_local = 1;"
+            ).results
+        ]
+        node_info.extend(nodes)
+
+    return node_info

snuba/admin/package.json

@@ -11,8 +11,9 @@
     "@codemirror/lang-sql": "^6.7.0",
     "@codemirror/state": "^6.4.1",
     "@codemirror/view": "^6.32.0",
-    "@mantine/core": "^6.0.15",
-    "@mantine/hooks": "^6.0.15",
+    "@mantine/core": "6.0.21",
+    "@mantine/dates": "6.0.21",
+    "@mantine/hooks": "6.0.21",
     "@mantine/prism": "^6.0.15",
     "@mantine/tiptap": "^6.0.15",
     "@sentry/react": "^7.88.0",
@@ -25,22 +26,24 @@
     "@tiptap/starter-kit": "^2.0.3",
     "@types/react": "^18.0.20",
     "@types/react-dom": "^18.2.6",
+    "dayjs": "^1.11.13",
     "jest-dom": "^4.0.0",
     "lowlight": "^2.9.0",
+    "mantine-react-table": "^1.3.4",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-select-event": "^5.5.1",
     "ts-loader": "^9.4.1",
     "typescript": "^4.8.3"
   },
   "devDependencies": {
-    "@emotion/react": "^11.11.1",
+    "@emotion/react": "^11.13.3",
     "@jest/globals": "^29.4.3",
     "@mantine/core": "^6.0.15",
     "@mantine/hooks": "^6.0.15",
     "@mantine/prism": "^6.0.15",
     "@mantine/tiptap": "^6.0.15",
-    "@tabler/icons-react": "^2.23.0",
+    "@tabler/icons-react": "^3.17.0",
     "@testing-library/react": "^14.0.0",
     "@testing-library/user-event": "^14.4.3",
     "@tiptap/extension-link": "^2.0.3",

snuba/admin/static/api_client.tsx

@@ -39,6 +39,7 @@ import { AllocationPolicy } from "SnubaAdmin/capacity_management/types";
 
 import { ReplayInstruction, Topic } from "SnubaAdmin/dead_letter_queue/types";
 import { AutoReplacementsBypassProjectsData } from "SnubaAdmin/auto_replacements_bypass_projects/types";
+import { ClickhouseNodeInfo } from "./database_clusters/types";
 
 interface Client {
   getSettings: () => Promise<Settings>;
@@ -60,6 +61,7 @@ interface Client {
   getDescriptions: () => Promise<ConfigDescriptions>;
   getAuditlog: () => Promise<ConfigChange[]>;
   getClickhouseNodes: () => Promise<[ClickhouseNodeData]>;
+  getClickhouseNodeInfo: () => Promise<[ClickhouseNodeInfo]>
   getSnubaDatasetNames: () => Promise<SnubaDatasetName[]>;
   getAllowedProjects: () => Promise<string[]>;
   executeSnQLQuery: (query: SnQLRequest) => Promise<any>;
@@ -207,6 +209,11 @@ function Client(): Client {
         });
     },
 
+    getClickhouseNodeInfo: () => {
+      const url = baseUrl + "clickhouse_node_info";
+      return fetch(url).then((resp) => resp.json());
+    },
+
     getSnubaDatasetNames: () => {
       const url = baseUrl + "snuba_datasets";
       return fetch(url).then((resp) => resp.json());

snuba/admin/static/data.tsx

@@ -16,6 +16,7 @@ import SnubaExplain from "SnubaAdmin/snuba_explain";
 import Welcome from "SnubaAdmin/welcome";
 import DeleteTool from "SnubaAdmin/delete_tool";
 import ViewCustomJobs from "SnubaAdmin/manual_jobs";
+import DatabaseClusters from "./database_clusters";
 
 const NAV_ITEMS = [
   { id: "overview", display: "🤿 Snuba Admin", component: Welcome },
@@ -100,6 +101,11 @@ const NAV_ITEMS = [
     display: "▶️ View/Run Custom Jobs",
     component: ViewCustomJobs,
   },
+  {
+    id: "database-clusters",
+    display: "🗂️ Database Clusters",
+    component: DatabaseClusters,
+  },
 ];
 
 export { NAV_ITEMS };

snuba/admin/static/database_clusters/index.tsx

@@ -0,0 +1,53 @@
+import { MantineReactTable, MRT_ColumnDef, useMantineReactTable } from "mantine-react-table";
+import React, { useMemo } from "react";
+import { useEffect, useState } from "react";
+import Client from "SnubaAdmin/api_client";
+import { ClickhouseNodeInfo } from "./types";
+
+function DatabaseClusters(props: { api: Client }) {
+    const [nodeData, setNodeData] = useState<ClickhouseNodeInfo[]>([]);
+
+    useEffect(() => {
+        props.api.getClickhouseNodeInfo().then((res) => {
+            setNodeData(res);
+        });
+    }, []);
+
+    const columns = useMemo<MRT_ColumnDef<ClickhouseNodeInfo>[]>(
+        () => [
+            {
+                accessorKey: 'cluster',
+                header: 'Cluster',
+            },
+            {
+                accessorFn: (row) => `${row.host_name} (${row.host_address})`,
+                header: 'Host',
+            },
+            {
+                accessorKey: 'shard',
+                header: 'Shard',
+            },
+            {
+                accessorKey: 'replica',
+                header: 'Replica',
+            },
+            {
+                accessorKey: 'version',
+                header: 'Version',
+                filterVariant: 'multi-select',
+            },
+        ],
+        [],
+    );
+
+    const table = useMantineReactTable({
+        columns,
+        data: nodeData,
+        initialState: { showColumnFilters: true },
+        enableFacetedValues: true,
+    });
+
+    return <MantineReactTable table={table} />;
+}
+
+export default DatabaseClusters

snuba/admin/static/database_clusters/types.tsx

@@ -0,0 +1,10 @@
+type ClickhouseNodeInfo = {
+    cluster: string,
+    host_name: string,
+    host_address: string,
+    shard: number,
+    replica: number,
+    version: string,
+};
+
+export { ClickhouseNodeInfo };

snuba/admin/tool_policies.py

@@ -33,6 +33,7 @@ class AdminTools(Enum):
     SNUBA_EXPLAIN = "snuba-explain"
     DELETE_TOOL = "delete_tool"
     MANUAL_JOBS = "view-jobs"
+    DATABASE_CLUSTERS = "database-clusters"
 
 
 DEVELOPER_TOOLS: set[AdminTools] = {AdminTools.SNQL_TO_SQL, AdminTools.QUERY_TRACING}

snuba/admin/views.py

@@ -22,6 +22,7 @@
     get_storages_with_allocation_policies,
 )
 from snuba.admin.clickhouse.common import InvalidCustomQuery
+from snuba.admin.clickhouse.database_clusters import get_node_info
 from snuba.admin.clickhouse.migration_checks import run_migration_checks_and_policies
 from snuba.admin.clickhouse.nodes import get_storage_info
 from snuba.admin.clickhouse.predefined_cardinality_analyzer_queries import (
@@ -1263,3 +1264,9 @@ def deletes_enabled() -> Response:
 @check_tool_perms(tools=[AdminTools.MANUAL_JOBS])
 def get_job_specs() -> Response:
     return make_response(jsonify(list_job_specs()), 200)
+
+
+@application.route("/clickhouse_node_info")
+@check_tool_perms(tools=[AdminTools.DATABASE_CLUSTERS])
+def clickhouse_node_info() -> Response:
+    return make_response(jsonify(get_node_info()), 200)