diff --git a/.github/workflows/typescript.yml b/.github/workflows/typescript.yml
index ac17dc3a..fd1fc150 100644
--- a/.github/workflows/typescript.yml
+++ b/.github/workflows/typescript.yml
@@ -64,3 +64,37 @@ jobs:
- name: Uninstall Yarn
if: always()
run: npm uninstall -g yarn
+
+ test-unit-dependency:
+ needs: check-typescript-changes
+ if: ${{ needs.check-typescript-changes.outputs.has_changes == 'yes' }}
+ name: Unit test
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ node-version: [20.7.0]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+
+ - name: Install Yarn
+ run: npm install -g yarn
+
+ - name: Install dependencies and run the tests
+ run: |
+ cd plugins/grafana-custom-plugins/grafana-dependency-plugin/
+ yarn install
+ yarn test --coverage
+
+ - name: Upload coverage reports to Codecov
+ uses: codecov/codecov-action@v3
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+
+ - name: Uninstall Yarn
+ if: always()
+ run: npm uninstall -g yarn
diff --git a/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json b/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json
index 53867799..fe724568 100644
--- a/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json
+++ b/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json
@@ -21,8 +21,8 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": 8,
- "iteration": 1682533463233,
+ "id": 6,
+ "iteration": 1695808205600,
"links": [],
"liveNow": false,
"panels": [
@@ -32,15 +32,55 @@
"uid": "PDEE91DDB90597936"
},
"gridPos": {
- "h": 35,
- "w": 23,
+ "h": 22,
+ "w": 10,
"x": 0,
"y": 0
},
+ "id": 4,
+ "options": {
+ "color": "yellow",
+ "groupByPodLabel": false,
+ "layerFour": true
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-clickhouse-datasource",
+ "uid": "PDEE91DDB90597936"
+ },
+ "meta": {
+ "builderOptions": {
+ "fields": [],
+ "limit": 100,
+ "mode": "list"
+ }
+ },
+ "queryType": "sql",
+ "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+ "refId": "A"
+ }
+ ],
+ "title": "Layer 4 Topology",
+ "type": "theia-grafana-dependency-plugin"
+ },
+ {
+ "datasource": {
+ "type": "grafana-clickhouse-datasource",
+ "uid": "PDEE91DDB90597936"
+ },
+ "gridPos": {
+ "h": 22,
+ "w": 10,
+ "x": 10,
+ "y": 0
+ },
"id": 2,
"options": {
"color": "yellow",
"groupByLabel": false,
+ "groupByPodLabel": false,
+ "layerFour": false,
"seriesCountSize": "sm",
"showSeriesCount": false,
"text": "Default value of text input option"
@@ -60,11 +100,11 @@
}
},
"queryType": "sql",
- "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+ "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
"refId": "A"
}
],
- "title": "Network Topology",
+ "title": "Layer 7 Topology",
"type": "theia-grafana-dependency-plugin"
}
],
@@ -102,6 +142,6 @@
"timezone": "",
"title": "network_topology_dashboard",
"uid": "yRVDEad4k",
- "version": 2,
+ "version": 3,
"weekStart": ""
}
diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh
old mode 100644
new mode 100755
index f8efb08f..53077b78
--- a/build/charts/theia/provisioning/datasources/create_table.sh
+++ b/build/charts/theia/provisioning/datasources/create_table.sh
@@ -80,6 +80,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
clusterUUID String,
egressName String,
egressIP String,
+ l7ProtocolName String,
+ httpVals String,
trusted UInt8 DEFAULT 0
) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
ORDER BY (timeInserted, flowEndSeconds);
diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql
new file mode 100644
index 00000000..8a89ea1a
--- /dev/null
+++ b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql
@@ -0,0 +1,7 @@
+--Alter table to drop new columns
+ALTER TABLE flows
+ DROP COLUMN l7ProtocolName,
+ DROP COLUMN httpVals;
+ALTER TABLE flows_local
+ DROP COLUMN l7ProtocolName,
+ DROP COLUMN httpVals;
diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql
new file mode 100644
index 00000000..4608bb81
--- /dev/null
+++ b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql
@@ -0,0 +1,7 @@
+--Alter table to add new columns
+ALTER TABLE flows
+ ADD COLUMN l7ProtocolName String,
+ ADD COLUMN httpVals String;
+ALTER TABLE flows_local
+ ADD COLUMN l7ProtocolName String,
+ ADD COLUMN httpVals String;
diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml
index b5c3f97b..cb7415bd 100644
--- a/build/yamls/flow-visibility.yml
+++ b/build/yamls/flow-visibility.yml
@@ -757,6 +757,22 @@ data:
ADD COLUMN aggType String,
ADD COLUMN direction String,
ADD COLUMN podName String;
+ 000006_0-7-0.down.sql: |
+ --Alter table to drop new columns
+ ALTER TABLE flows
+ DROP COLUMN l7ProtocolName,
+ DROP COLUMN httpVals;
+ ALTER TABLE flows_local
+ DROP COLUMN l7ProtocolName,
+ DROP COLUMN httpVals;
+ 000006_0-7-0.up.sql: |
+ --Alter table to add new columns
+ ALTER TABLE flows
+ ADD COLUMN l7ProtocolName String,
+ ADD COLUMN httpVals String;
+ ALTER TABLE flows_local
+ ADD COLUMN l7ProtocolName String,
+ ADD COLUMN httpVals String;
create_table.sh: |
#!/usr/bin/env bash
@@ -831,6 +847,8 @@ data:
clusterUUID String,
egressName String,
egressIP String,
+ l7ProtocolName String,
+ httpVals String,
trusted UInt8 DEFAULT 0
) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}')
ORDER BY (timeInserted, flowEndSeconds);
@@ -3532,8 +3550,8 @@ data:
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": 8,
- "iteration": 1682533463233,
+ "id": 6,
+ "iteration": 1695808205600,
"links": [],
"liveNow": false,
"panels": [
@@ -3543,15 +3561,55 @@ data:
"uid": "PDEE91DDB90597936"
},
"gridPos": {
- "h": 35,
- "w": 23,
+ "h": 22,
+ "w": 10,
"x": 0,
"y": 0
},
+ "id": 4,
+ "options": {
+ "color": "yellow",
+ "groupByPodLabel": false,
+ "layerFour": true
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "grafana-clickhouse-datasource",
+ "uid": "PDEE91DDB90597936"
+ },
+ "meta": {
+ "builderOptions": {
+ "fields": [],
+ "limit": 100,
+ "mode": "list"
+ }
+ },
+ "queryType": "sql",
+ "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+ "refId": "A"
+ }
+ ],
+ "title": "Layer 4 Topology",
+ "type": "theia-grafana-dependency-plugin"
+ },
+ {
+ "datasource": {
+ "type": "grafana-clickhouse-datasource",
+ "uid": "PDEE91DDB90597936"
+ },
+ "gridPos": {
+ "h": 22,
+ "w": 10,
+ "x": 10,
+ "y": 0
+ },
"id": 2,
"options": {
"color": "yellow",
"groupByLabel": false,
+ "groupByPodLabel": false,
+ "layerFour": false,
"seriesCountSize": "sm",
"showSeriesCount": false,
"text": "Default value of text input option"
@@ -3571,11 +3629,11 @@ data:
}
},
"queryType": "sql",
- "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
+ "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC",
"refId": "A"
}
],
- "title": "Network Topology",
+ "title": "Layer 7 Topology",
"type": "theia-grafana-dependency-plugin"
}
],
@@ -3613,7 +3671,7 @@ data:
"timezone": "",
"title": "network_topology_dashboard",
"uid": "yRVDEad4k",
- "version": 2,
+ "version": 3,
"weekStart": ""
}
networkpolicy_dashboard.json: |
@@ -7329,6 +7387,10 @@ spec:
path: migrators/000005_0-6-0.down.sql
- key: 000005_0-6-0.up.sql
path: migrators/000005_0-6-0.up.sql
+ - key: 000006_0-7-0.down.sql
+ path: migrators/000006_0-7-0.down.sql
+ - key: 000006_0-7-0.up.sql
+ path: migrators/000006_0-7-0.up.sql
name: clickhouse-mounted-configmap
name: clickhouse-configmap-volume
- emptyDir:
diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md
index 0380d926..b055fb9c 100644
--- a/docs/network-flow-visibility.md
+++ b/docs/network-flow-visibility.md
@@ -867,12 +867,19 @@ Mouse out or click on the background will bring all the traffic back.
#### Network Topology Dashboard
-Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic.
-Pods are visually grouped by Node. Each arrow points from the source to the
-destination, and is labelled by the cumulative amount of data transmitted in
-the selected time range.
+Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic
+at layer 4, and IP-to-IP traffic at layer 7.
-
+In the layer 4 visualization, Pods are visually grouped by Node. Each arrow
+points from the source to the destination, and is labelled by the cumulative
+amount of data transmitted in the selected time range.
+
+In the layer 7 visualization, Pods and IPs with ports are used to depict flows,
+with each arrow representing the data from a singular set of HTTP values. The
+color of each arrow represents the status of the http request, and is labelled
+by the content length of the message or data.
+
+
By editing the panel, users can group Pods by label, allowing Pod squares in
the diagram to represent a set of Pods with the same label value. It is also
diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc
new file mode 100644
index 00000000..15ae260a
--- /dev/null
+++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc
@@ -0,0 +1,13 @@
+/*
+ * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️
+ *
+ * In order to extend the configuration follow the steps in
+ * https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/extend-configurations#extend-the-eslint-config
+ */
+ {
+ "extends": ["@grafana/eslint-config"],
+ "root": true,
+ "rules": {
+ "react/prop-types": "off"
+ }
+}
diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js
new file mode 100644
index 00000000..4afe885a
--- /dev/null
+++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js
@@ -0,0 +1,16 @@
+/*
+ * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️
+ *
+ * In order to extend the configuration follow the steps in .config/README.md
+ */
+
+module.exports = {
+ "endOfLine": "auto",
+ "printWidth": 120,
+ "trailingComma": "es5",
+ "semi": true,
+ "jsxSingleQuote": false,
+ "singleQuote": true,
+ "useTabs": false,
+ "tabWidth": 2
+};
diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile
new file mode 100644
index 00000000..40537a54
--- /dev/null
+++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile
@@ -0,0 +1,16 @@
+ARG grafana_version=latest
+ARG grafana_image=grafana-enterprise
+
+FROM grafana/${grafana_image}:${grafana_version}
+
+# Make it as simple as possible to access the grafana instance for development purposes
+# Do NOT enable these settings in a public facing / production grafana instance
+ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin"
+ENV GF_AUTH_ANONYMOUS_ENABLED "true"
+ENV GF_AUTH_BASIC_ENABLED "false"
+# Set development mode so plugins can be loaded without the need to sign
+ENV GF_DEFAULT_APP_MODE "development"
+
+# Inject livereload script into grafana index.html
+USER root
+RUN sed -i 's/<\/body><\/html>/