diff --git a/.github/workflows/typescript.yml b/.github/workflows/typescript.yml index ac17dc3a..fd1fc150 100644 --- a/.github/workflows/typescript.yml +++ b/.github/workflows/typescript.yml @@ -64,3 +64,37 @@ jobs: - name: Uninstall Yarn if: always() run: npm uninstall -g yarn + + test-unit-dependency: + needs: check-typescript-changes + if: ${{ needs.check-typescript-changes.outputs.has_changes == 'yes' }} + name: Unit test + strategy: + matrix: + os: [ubuntu-latest] + node-version: [20.7.0] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: Install Yarn + run: npm install -g yarn + + - name: Install dependencies and run the tests + run: | + cd plugins/grafana-custom-plugins/grafana-dependency-plugin/ + yarn install + yarn test --coverage + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Uninstall Yarn + if: always() + run: npm uninstall -g yarn diff --git a/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json b/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json index 53867799..fe724568 100644 --- a/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json +++ b/build/charts/theia/provisioning/dashboards/network_topology_dashboard.json @@ -21,8 +21,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 8, - "iteration": 1682533463233, + "id": 6, + "iteration": 1695808205600, "links": [], "liveNow": false, "panels": [ @@ -32,15 +32,55 @@ "uid": "PDEE91DDB90597936" }, "gridPos": { - "h": 35, - "w": 23, + "h": 22, + "w": 10, "x": 0, "y": 0 }, + "id": 4, + "options": { + "color": "yellow", + "groupByPodLabel": false, + "layerFour": true + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", + "refId": "A" + } + ], + "title": "Layer 4 Topology", + "type": "theia-grafana-dependency-plugin" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "gridPos": { + "h": 22, + "w": 10, + "x": 10, + "y": 0 + }, "id": 2, "options": { "color": "yellow", "groupByLabel": false, + "groupByPodLabel": false, + "layerFour": false, "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" @@ -60,11 +100,11 @@ } }, "queryType": "sql", - "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", + "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", "refId": "A" } ], - "title": "Network Topology", + "title": "Layer 7 Topology", "type": "theia-grafana-dependency-plugin" } ], @@ -102,6 +142,6 @@ "timezone": "", "title": "network_topology_dashboard", "uid": "yRVDEad4k", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/build/charts/theia/provisioning/datasources/create_table.sh b/build/charts/theia/provisioning/datasources/create_table.sh old mode 100644 new mode 100755 index f8efb08f..53077b78 --- a/build/charts/theia/provisioning/datasources/create_table.sh +++ b/build/charts/theia/provisioning/datasources/create_table.sh @@ -80,6 +80,8 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL clusterUUID String, egressName String, egressIP String, + l7ProtocolName String, + httpVals String, trusted UInt8 DEFAULT 0 ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeInserted, flowEndSeconds); diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql new file mode 100644 index 00000000..8a89ea1a --- /dev/null +++ b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.down.sql @@ -0,0 +1,7 @@ +--Alter table to drop new columns +ALTER TABLE flows + DROP COLUMN l7ProtocolName, + DROP COLUMN httpVals; +ALTER TABLE flows_local + DROP COLUMN l7ProtocolName, + DROP COLUMN httpVals; diff --git a/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql new file mode 100644 index 00000000..4608bb81 --- /dev/null +++ b/build/charts/theia/provisioning/datasources/migrators/000006_0-7-0.up.sql @@ -0,0 +1,7 @@ +--Alter table to add new columns +ALTER TABLE flows + ADD COLUMN l7ProtocolName String, + ADD COLUMN httpVals String; +ALTER TABLE flows_local + ADD COLUMN l7ProtocolName String, + ADD COLUMN httpVals String; diff --git a/build/yamls/flow-visibility.yml b/build/yamls/flow-visibility.yml index b5c3f97b..cb7415bd 100644 --- a/build/yamls/flow-visibility.yml +++ b/build/yamls/flow-visibility.yml @@ -757,6 +757,22 @@ data: ADD COLUMN aggType String, ADD COLUMN direction String, ADD COLUMN podName String; + 000006_0-7-0.down.sql: | + --Alter table to drop new columns + ALTER TABLE flows + DROP COLUMN l7ProtocolName, + DROP COLUMN httpVals; + ALTER TABLE flows_local + DROP COLUMN l7ProtocolName, + DROP COLUMN httpVals; + 000006_0-7-0.up.sql: | + --Alter table to add new columns + ALTER TABLE flows + ADD COLUMN l7ProtocolName String, + ADD COLUMN httpVals String; + ALTER TABLE flows_local + ADD COLUMN l7ProtocolName String, + ADD COLUMN httpVals String; create_table.sh: | #!/usr/bin/env bash @@ -831,6 +847,8 @@ data: clusterUUID String, egressName String, egressIP String, + l7ProtocolName String, + httpVals String, trusted UInt8 DEFAULT 0 ) engine=ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') ORDER BY (timeInserted, flowEndSeconds); @@ -3532,8 +3550,8 @@ data: "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 8, - "iteration": 1682533463233, + "id": 6, + "iteration": 1695808205600, "links": [], "liveNow": false, "panels": [ @@ -3543,15 +3561,55 @@ data: "uid": "PDEE91DDB90597936" }, "gridPos": { - "h": 35, - "w": 23, + "h": 22, + "w": 10, "x": 0, "y": 0 }, + "id": 4, + "options": { + "color": "yellow", + "groupByPodLabel": false, + "layerFour": true + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "meta": { + "builderOptions": { + "fields": [], + "limit": 100, + "mode": "list" + } + }, + "queryType": "sql", + "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND ( destinationPodName != '' OR sourcePodName != '' )\nAND octetDeltaCount != 0\nAND httpVals == ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", + "refId": "A" + } + ], + "title": "Layer 4 Topology", + "type": "theia-grafana-dependency-plugin" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "gridPos": { + "h": 22, + "w": 10, + "x": 10, + "y": 0 + }, "id": 2, "options": { "color": "yellow", "groupByLabel": false, + "groupByPodLabel": false, + "layerFour": false, "seriesCountSize": "sm", "showSeriesCount": false, "text": "Default value of text input option" @@ -3571,11 +3629,11 @@ data: } }, "queryType": "sql", - "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, octetDeltaCount FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodName != ''\nAND sourcePodName != ''\nAND octetDeltaCount != 0\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", + "rawSql": "SELECT sourcePodName, sourcePodLabels, sourcePodNamespace, sourceNodeName, sourceIP, sourceTransportPort, destinationPodName, destinationPodLabels, destinationNodeName, destinationServicePortName, destinationIP, destinationTransportPort, octetDeltaCount, httpVals FROM flows\nWHERE sourcePodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND destinationPodNamespace NOT IN ('kube-system', 'flow-visibility', 'flow-aggregator')\nAND octetDeltaCount != 0\nAND httpVals != ''\nAND $__timeFilter(flowEndSeconds)\nORDER BY flowEndSeconds DESC", "refId": "A" } ], - "title": "Network Topology", + "title": "Layer 7 Topology", "type": "theia-grafana-dependency-plugin" } ], @@ -3613,7 +3671,7 @@ data: "timezone": "", "title": "network_topology_dashboard", "uid": "yRVDEad4k", - "version": 2, + "version": 3, "weekStart": "" } networkpolicy_dashboard.json: | @@ -7329,6 +7387,10 @@ spec: path: migrators/000005_0-6-0.down.sql - key: 000005_0-6-0.up.sql path: migrators/000005_0-6-0.up.sql + - key: 000006_0-7-0.down.sql + path: migrators/000006_0-7-0.down.sql + - key: 000006_0-7-0.up.sql + path: migrators/000006_0-7-0.up.sql name: clickhouse-mounted-configmap name: clickhouse-configmap-volume - emptyDir: diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md index 0380d926..b055fb9c 100644 --- a/docs/network-flow-visibility.md +++ b/docs/network-flow-visibility.md @@ -867,12 +867,19 @@ Mouse out or click on the background will bring all the traffic back. #### Network Topology Dashboard -Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic. -Pods are visually grouped by Node. Each arrow points from the source to the -destination, and is labelled by the cumulative amount of data transmitted in -the selected time range. +Network Topology Dashboard visualizes Pod-to-Pod and Pod-to-Service traffic +at layer 4, and IP-to-IP traffic at layer 7. -Network Topology Dashboard service dependency graph +In the layer 4 visualization, Pods are visually grouped by Node. Each arrow +points from the source to the destination, and is labelled by the cumulative +amount of data transmitted in the selected time range. + +In the layer 7 visualization, Pods and IPs with ports are used to depict flows, +with each arrow representing the data from a singular set of HTTP values. The +color of each arrow represents the status of the http request, and is labelled +by the content length of the message or data. + +Network Topology Dashboard By editing the panel, users can group Pods by label, allowing Pod squares in the diagram to represent a set of Pods with the same label value. It is also diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc new file mode 100644 index 00000000..15ae260a --- /dev/null +++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.eslintrc @@ -0,0 +1,13 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in + * https://grafana.com/developers/plugin-tools/create-a-plugin/extend-a-plugin/extend-configurations#extend-the-eslint-config + */ + { + "extends": ["@grafana/eslint-config"], + "root": true, + "rules": { + "react/prop-types": "off" + } +} diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js new file mode 100644 index 00000000..4afe885a --- /dev/null +++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/.prettierrc.js @@ -0,0 +1,16 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in .config/README.md + */ + +module.exports = { + "endOfLine": "auto", + "printWidth": 120, + "trailingComma": "es5", + "semi": true, + "jsxSingleQuote": false, + "singleQuote": true, + "useTabs": false, + "tabWidth": 2 +}; diff --git a/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile new file mode 100644 index 00000000..40537a54 --- /dev/null +++ b/plugins/grafana-custom-plugins/grafana-dependency-plugin/.config/Dockerfile @@ -0,0 +1,16 @@ +ARG grafana_version=latest +ARG grafana_image=grafana-enterprise + +FROM grafana/${grafana_image}:${grafana_version} + +# Make it as simple as possible to access the grafana instance for development purposes +# Do NOT enable these settings in a public facing / production grafana instance +ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin" +ENV GF_AUTH_ANONYMOUS_ENABLED "true" +ENV GF_AUTH_BASIC_ENABLED "false" +# Set development mode so plugins can be loaded without the need to sign +ENV GF_DEFAULT_APP_MODE "development" + +# Inject livereload script into grafana index.html +USER root +RUN sed -i 's/<\/body><\/html>/