Network Topology Dashboard Update

This PR updates the Network Topology dashboard to include visualizations for multiple networking layers. It also updates the legends of both graphs to improve readability. Signed-off-by: Dhruv-J <[email protected]>
antrea-io · Nov 6, 2023 · 0b94865 · 0b94865
1 parent 79010a7
commit 0b94865
Show file tree

Hide file tree

Showing 33 changed files with 7,219 additions and 5,760 deletions.
diff --git a/.github/workflows/typescript.yml b/.github/workflows/typescript.yml
@@ -64,3 +64,37 @@ jobs:
     - name: Uninstall Yarn
       if: always()
       run: npm uninstall -g yarn
+
+  test-unit-dependency:
+    needs: check-typescript-changes
+    if: ${{ needs.check-typescript-changes.outputs.has_changes == 'yes' }}
+    name: Unit test
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [20.7.0]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Yarn
+      run: npm install -g yarn
+
+    - name: Install dependencies and run the tests
+      run: |
+        cd plugins/grafana-custom-plugins/grafana-dependency-plugin/
+        yarn install
+        yarn test --coverage
+
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
+    - name: Uninstall Yarn
+      if: always()
+      run: npm uninstall -g yarn
diff --git a/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json b/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json
@@ -21,8 +21,8 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 8,
-  "iteration": 1682533463233,
+  "id": 6,
+  "iteration": 1695808205600,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -32,15 +32,55 @@
         "uid": "PDEE91DDB90597936"
       },
       "gridPos": {
-        "h": 35,
-        "w": 23,
+        "h": 22,
+        "w": 10,
         "x": 0,
         "y": 0
       },
+      "id": 4,
+      "options": {
+        "color": "yellow",
+        "groupByPodLabel": false,
+        "layerFour": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "grafana-clickhouse-datasource",
+            "uid": "PDEE91DDB90597936"
+          },
+          "meta": {
+            "builderOptions": {
+              "fields": [],
+              "limit": 100,
+              "mode": "list"
+            }
+          },
+          "queryType": "sql",
+          "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+          "refId": "A"
+        }
+      ],
+      "title": "Layer 4 Topology",
+      "type": "theia-grafana-dependency-plugin"
+    },
+    {
+      "datasource": {
+        "type": "grafana-clickhouse-datasource",
+        "uid": "PDEE91DDB90597936"
+      },
+      "gridPos": {
+        "h": 22,
+        "w": 10,
+        "x": 10,
+        "y": 0
+      },
       "id": 2,
       "options": {
         "color": "yellow",
         "groupByLabel": false,
+        "groupByPodLabel": false,
+        "layerFour": false,
         "seriesCountSize": "sm",
         "showSeriesCount": false,
         "text": "Default value of text input option"
@@ -60,11 +100,11 @@
             }
           },
           "queryType": "sql",
-          "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+          "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
           "refId": "A"
         }
       ],
-      "title": "Network Topology",
+      "title": "Layer 7 Topology",
       "type": "theia-grafana-dependency-plugin"
     }
   ],
@@ -102,6 +142,6 @@
   "timezone": "",
   "title": "network_topology_dashboard",
   "uid": "yRVDEad4k",
-  "version": 2,
+  "version": 3,
   "weekStart": ""
 }
diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh
@@ -80,6 +80,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         clusterUUID String,
         egressName String,
         egressIP String,
+        l7ProtocolName String,
+        httpVals String,
         trusted UInt8 DEFAULT 0
     ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
     ORDER BY (timeInserted, flowEndSeconds);

diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql
@@ -0,0 +1,7 @@
+--Alter table to drop new columns
+ALTER TABLE flows
+    DROP COLUMN l7ProtocolName,
+    DROP COLUMN httpVals;
+ALTER TABLE flows_local
+    DROP COLUMN l7ProtocolName,
+    DROP COLUMN httpVals;
diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql
@@ -0,0 +1,7 @@
+--Alter table to add new columns
+ALTER TABLE flows
+    ADD COLUMN l7ProtocolName String,
+    ADD COLUMN httpVals String;
+ALTER TABLE flows_local
+    ADD COLUMN l7ProtocolName String,
+    ADD COLUMN httpVals String;
diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml
@@ -757,6 +757,22 @@ data:
         ADD COLUMN aggType String,
         ADD COLUMN direction String,
         ADD COLUMN podName String;
+  000006_0-7-0.down.sql: |
+    --Alter table to drop new columns
+    ALTER TABLE flows
+        DROP COLUMN l7ProtocolName,
+        DROP COLUMN httpVals;
+    ALTER TABLE flows_local
+        DROP COLUMN l7ProtocolName,
+        DROP COLUMN httpVals;
+  000006_0-7-0.up.sql: |
+    --Alter table to add new columns
+    ALTER TABLE flows
+        ADD COLUMN l7ProtocolName String,
+        ADD COLUMN httpVals String;
+    ALTER TABLE flows_local
+        ADD COLUMN l7ProtocolName String,
+        ADD COLUMN httpVals String;
   create_table.sh: |
     #!/usr/bin/env bash
 
@@ -831,6 +847,8 @@ data:
             clusterUUID String,
             egressName String,
             egressIP String,
+            l7ProtocolName String,
+            httpVals String,
             trusted UInt8 DEFAULT 0
         ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
         ORDER BY (timeInserted, flowEndSeconds);
@@ -3532,8 +3550,8 @@ data:
       "editable": true,
       "fiscalYearStartMonth": 0,
       "graphTooltip": 0,
-      "id": 8,
-      "iteration": 1682533463233,
+      "id": 6,
+      "iteration": 1695808205600,
       "links": [],
       "liveNow": false,
       "panels": [
@@ -3543,15 +3561,55 @@ data:
             "uid": "PDEE91DDB90597936"
           },
           "gridPos": {
-            "h": 35,
-            "w": 23,
+            "h": 22,
+            "w": 10,
             "x": 0,
             "y": 0
           },
+          "id": 4,
+          "options": {
+            "color": "yellow",
+            "groupByPodLabel": false,
+            "layerFour": true
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "grafana-clickhouse-datasource",
+                "uid": "PDEE91DDB90597936"
+              },
+              "meta": {
+                "builderOptions": {
+                  "fields": [],
+                  "limit": 100,
+                  "mode": "list"
+                }
+              },
+              "queryType": "sql",
+              "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+              "refId": "A"
+            }
+          ],
+          "title": "Layer 4 Topology",
+          "type": "theia-grafana-dependency-plugin"
+        },
+        {
+          "datasource": {
+            "type": "grafana-clickhouse-datasource",
+            "uid": "PDEE91DDB90597936"
+          },
+          "gridPos": {
+            "h": 22,
+            "w": 10,
+            "x": 10,
+            "y": 0
+          },
           "id": 2,
           "options": {
             "color": "yellow",
             "groupByLabel": false,
+            "groupByPodLabel": false,
+            "layerFour": false,
             "seriesCountSize": "sm",
             "showSeriesCount": false,
             "text": "Default value of text input option"
@@ -3571,11 +3629,11 @@ data:
                 }
               },
               "queryType": "sql",
-              "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+              "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
               "refId": "A"
             }
           ],
-          "title": "Network Topology",
+          "title": "Layer 7 Topology",
           "type": "theia-grafana-dependency-plugin"
         }
       ],
@@ -3613,7 +3671,7 @@ data:
       "timezone": "",
       "title": "network_topology_dashboard",
       "uid": "yRVDEad4k",
-      "version": 2,
+      "version": 3,
       "weekStart": ""
     }
   networkpolicy_dashboard.json: |
@@ -7329,6 +7387,10 @@ spec:
               path: migrators/000005_0-6-0.down.sql
             - key: 000005_0-6-0.up.sql
               path: migrators/000005_0-6-0.up.sql
+            - key: 000006_0-7-0.down.sql
+              path: migrators/000006_0-7-0.down.sql
+            - key: 000006_0-7-0.up.sql
+              path: migrators/000006_0-7-0.up.sql
             name: clickhouse-mounted-configmap
           name: clickhouse-configmap-volume
         - emptyDir:

diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md
@@ -867,12 +867,19 @@ Mouse out or click on the background will bring all the traffic back.
 
 #### Network Topology Dashboard
 
-Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic.
-Pods are visually grouped by Node. Each arrow points from the source to the
-destination, and is labelled by the cumulative amount of data transmitted in
-the selected time range.
+Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic
+at layer 4, and IP-to-IP traffic at layer 7.
 
-<img src="https://downloads.antrea.io/static/04132023/flow-visibility-network-topology-0.png" width="400" alt="Network Topology Dashboard service dependency graph">
+In the layer 4 visualization, Pods are visually grouped by Node. Each arrow
+points from the source to the destination, and is labelled by the cumulative
+amount of data transmitted in the selected time range.
+
+In the layer 7 visualization, Pods and IPs with ports are used to depict flows,
+with each arrow representing the data from a singular set of HTTP values. The
+color of each arrow represents the status of the http request, and is labelled
+by the content length of the message or data.
+
+<img src="https://user-images.githubusercontent.com/10016630/271056005-7dfb4072-f64f-4d1e-b4b9-4f81eb7a51fa.png" width="400" alt="Network Topology Dashboard">
 
 By editing the panel, users can group Pods by label, allowing Pod squares in
 the diagram to represent a set of Pods with the same label value. It is also

diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc
@@ -0,0 +1,13 @@
+/*
+ * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️
+ *
+ * In order to extend the configuration follow the steps in
+ * https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/extend-configurations#extend-the-eslint-config
+ */
+ {
+  "extends": ["@grafana/eslint-config"],
+  "root": true,
+  "rules": {
+    "react/prop-types": "off"
+  }
+}
diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js
@@ -0,0 +1,16 @@
+/*
+ * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️
+ *
+ * In order to extend the configuration follow the steps in .config/README.md
+ */
+
+module.exports = {
+  "endOfLine": "auto",
+  "printWidth": 120,
+  "trailingComma": "es5",
+  "semi": true,
+  "jsxSingleQuote": false,
+  "singleQuote": true,
+  "useTabs": false,
+  "tabWidth": 2
+};
diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile
@@ -0,0 +1,16 @@
+ARG grafana_version=latest
+ARG grafana_image=grafana-enterprise
+
+FROM grafana/${grafana_image}:${grafana_version}
+
+# Make it as simple as possible to access the grafana instance for development purposes
+# Do NOT enable these settings in a public facing / production grafana instance
+ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin"
+ENV GF_AUTH_ANONYMOUS_ENABLED "true"
+ENV GF_AUTH_BASIC_ENABLED "false"
+# Set development mode so plugins can be loaded without the need to sign
+ENV GF_DEFAULT_APP_MODE "development"
+
+# Inject livereload script into grafana index.html
+USER root
+RUN sed -i 's/<\/body><\/html>/<script src=\"http:\/\/localhost:35729\/livereload.js\"><\/script><\/body><\/html>/g' /usr/share/grafana/public/views/index.html