From 28c7a803093247530454ed410b9fb6ed60f208c7 Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Sun, 25 Aug 2024 04:13:42 +0200 Subject: [PATCH 1/3] Added metrics to tests Fix liteprotocoltester docker files with libnegentropy COPY docker compose with waku-sim simulation now having test performance dashboard and localhost:3033 Mention dashboard in Readme --- apps/liteprotocoltester/.env | 2 +- .../Dockerfile.liteprotocoltester | 2 + .../Dockerfile.liteprotocoltester.compile | 1 + .../Dockerfile.liteprotocoltester.copy | 35 - apps/liteprotocoltester/README.md | 4 + .../docker-compose-on-simularor.yml | 74 +- apps/liteprotocoltester/docker-compose.yml | 9 +- .../lightpush_publisher.nim | 6 +- .../liteprotocoltester/liteprotocoltester.nim | 7 +- apps/liteprotocoltester/lpt_metrics.nim | 30 + .../liter-protocol-test-monitoring.json | 1091 ++++ .../dashboards/nwaku-monitoring.json | 5776 ----------------- .../monitoring/configuration/datasources.yaml | 4 +- .../monitoring/configuration/grafana.ini | 4 +- .../configuration/pg-exporter-queries.yml | 284 - .../configuration/postgres-exporter.yml | 9 - .../monitoring/prometheus-config.yml | 12 +- apps/liteprotocoltester/run_service_node.sh | 1 + apps/liteprotocoltester/run_tester_node.sh | 8 +- apps/liteprotocoltester/statistics.nim | 30 +- 20 files changed, 1230 insertions(+), 6159 deletions(-) delete mode 100644 apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy create mode 100644 apps/liteprotocoltester/lpt_metrics.nim create mode 100644 apps/liteprotocoltester/monitoring/configuration/dashboards/liter-protocol-test-monitoring.json delete mode 100644 apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json delete mode 100644 apps/liteprotocoltester/monitoring/configuration/pg-exporter-queries.yml delete mode 100644 apps/liteprotocoltester/monitoring/configuration/postgres-exporter.yml diff --git a/apps/liteprotocoltester/.env b/apps/liteprotocoltester/.env index b389cc3cfc..a65106e737 100644 --- a/apps/liteprotocoltester/.env +++ b/apps/liteprotocoltester/.env @@ -4,7 +4,7 @@ START_PUBLISHING_AFTER=10 NUM_MESSAGES=0 # 0 for infinite number of messages -DELAY_MESSAGES=8000 +DELAY_MESSAGES=188 # ms delay between messages diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester b/apps/liteprotocoltester/Dockerfile.liteprotocoltester index 11e37078d0..d6d4e67f39 100644 --- a/apps/liteprotocoltester/Dockerfile.liteprotocoltester +++ b/apps/liteprotocoltester/Dockerfile.liteprotocoltester @@ -29,6 +29,8 @@ COPY build/liteprotocoltester /usr/bin/ COPY apps/liteprotocoltester/run_tester_node.sh /usr/bin/ + COPY ./libnegentropy.so /usr/lib/ + ENTRYPOINT ["/usr/bin/run_tester_node.sh", "/usr/bin/liteprotocoltester"] diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile index 6e3184c9ac..d5b45ee2d5 100644 --- a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile +++ b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile @@ -51,6 +51,7 @@ # Copy migration scripts for DB upgrades COPY --from=nim-build /app/migrations/ /app/migrations/ + COPY --from=nim-build /app/libnegentropy.so /usr/lib/ ENTRYPOINT ["/usr/bin/liteprotocoltester"] diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy deleted file mode 100644 index bfff8bae70..0000000000 --- a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy +++ /dev/null @@ -1,35 +0,0 @@ - # TESTING IMAGE -------------------------------------------------------------- - - ## NOTICE: This is a short cut build file for ubuntu users who compiles nwaku in ubuntu distro. - ## This is used for faster turnaround time for testing the compiled binary. - ## Prerequisites: compiled liteprotocoltester binary in build/ directory - - FROM ubuntu:noble AS prod - - LABEL maintainer="jakub@status.im" - LABEL source="https://github.com/waku-org/nwaku" - LABEL description="Lite Protocol Tester: Waku light-client" - LABEL commit="unknown" - LABEL version="unknown" - - # DevP2P, LibP2P, and JSON RPC ports - EXPOSE 30303 60000 8545 - - # Referenced in the binary - RUN apt-get update && apt-get install -y --no-install-recommends \ - libgcc1 \ - libpcre3 \ - libpq-dev \ - wget \ - iproute2 \ - && rm -rf /var/lib/apt/lists/* - - # Fix for 'Error loading shared library libpcre.so.3: No such file or directory' - RUN ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3 - - COPY build/liteprotocoltester /usr/bin/ - - ENTRYPOINT ["/usr/bin/liteprotocoltester"] - - # # By default just show help if called without arguments - CMD ["--help"] diff --git a/apps/liteprotocoltester/README.md b/apps/liteprotocoltester/README.md index 719616d630..1e7d45f5cf 100644 --- a/apps/liteprotocoltester/README.md +++ b/apps/liteprotocoltester/README.md @@ -96,6 +96,10 @@ docker compose -f docker-compose-on-simularor.yml logs -f receivernode - Notice there is a configurable wait before start publishing messages as it is noticed time is needed for the service nodes to get connected to full nodes from simulator - light clients will print report on their and the connected service node's connectivity to the network in every 20 secs. +#### Test monitoring + +Navigate to http://localhost:3033 to see the lite-protocol-tester dashboard. + ### Phase 3 > Run independently on a chosen waku fleet diff --git a/apps/liteprotocoltester/docker-compose-on-simularor.yml b/apps/liteprotocoltester/docker-compose-on-simularor.yml index 24145801fe..e6002376dc 100644 --- a/apps/liteprotocoltester/docker-compose-on-simularor.yml +++ b/apps/liteprotocoltester/docker-compose-on-simularor.yml @@ -62,7 +62,7 @@ services: image: waku.liteprotocoltester:latest build: context: ../.. - dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy + dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester deploy: replicas: ${NUM_PUBLISHER_NODES:-3} # ports: @@ -84,13 +84,12 @@ services: - *rln_env - *test_running_conditions volumes: - - ./run_tester_node.sh:/opt/run_tester_node.sh:Z - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z - ./rln_tree:/etc/rln_tree/:Z - ./keystore:/keystore:Z entrypoint: sh command: - - /opt/run_tester_node.sh + - /usr/bin/run_tester_node.sh - /usr/bin/liteprotocoltester - SENDER - waku-sim @@ -139,7 +138,7 @@ services: image: waku.liteprotocoltester:latest build: context: ../.. - dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy + dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester deploy: replicas: ${NUM_RECEIVER_NODES:-1} # ports: @@ -161,13 +160,12 @@ services: - *rln_env - *test_running_conditions volumes: - - ./run_tester_node.sh:/opt/run_tester_node.sh:Z - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z - ./rln_tree:/etc/rln_tree/:Z - ./keystore:/keystore:Z entrypoint: sh command: - - /opt/run_tester_node.sh + - /usr/bin/run_tester_node.sh - /usr/bin/liteprotocoltester - RECEIVER - waku-sim @@ -180,34 +178,44 @@ services: networks: - waku-simulator_simulation - ## We have prometheus and grafana defined in waku-simulator already - # prometheus: - # image: docker.io/prom/prometheus:latest - # volumes: - # - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z - # command: - # - --config.file=/etc/prometheus/prometheus.yml - # ports: - # - 127.0.0.1:9090:9090 - # depends_on: - # - servicenode + # We have prometheus and grafana defined in waku-simulator already + prometheus: + image: docker.io/prom/prometheus:latest + volumes: + - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.listen-address=:9099 + # ports: + # - 127.0.0.1:9090:9090 + restart: on-failure:5 + depends_on: + - filter-service + - lightpush-service + - publishernode + - receivernode + networks: + - waku-simulator_simulation - # grafana: - # image: docker.io/grafana/grafana:latest - # env_file: - # - ./monitoring/configuration/grafana-plugins.env - # volumes: - # - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z - # - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z - # - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z - # - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z - # - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z - # - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z - # - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z - # ports: - # - 0.0.0.0:3000:3000 - # depends_on: - # - prometheus + grafana: + image: docker.io/grafana/grafana:latest + env_file: + - ./monitoring/configuration/grafana-plugins.env + volumes: + - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z + - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z + - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z + - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z + - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z + - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z + - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z + ports: + - 0.0.0.0:3033:3033 + restart: on-failure:5 + depends_on: + - prometheus + networks: + - waku-simulator_simulation configs: cfg_tester_node.toml: diff --git a/apps/liteprotocoltester/docker-compose.yml b/apps/liteprotocoltester/docker-compose.yml index c13bdd973e..32f67fea1a 100644 --- a/apps/liteprotocoltester/docker-compose.yml +++ b/apps/liteprotocoltester/docker-compose.yml @@ -59,7 +59,7 @@ services: image: waku.liteprotocoltester:latest build: context: ../.. - dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy + dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester ports: # - 30304:30304/tcp # - 30304:30304/udp @@ -79,13 +79,12 @@ services: - *rln_env - *test_running_conditions volumes: - - ./run_tester_node.sh:/opt/run_tester_node.sh:Z - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z - ./rln_tree:/etc/rln_tree/:Z - ./keystore:/keystore:Z entrypoint: sh command: - - /opt/run_tester_node.sh + - /usr/bin/run_tester_node.sh - /usr/bin/liteprotocoltester - SENDER - servicenode @@ -99,7 +98,7 @@ services: image: waku.liteprotocoltester:latest build: context: ../.. - dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy + dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester ports: # - 30304:30304/tcp # - 30304:30304/udp @@ -125,7 +124,7 @@ services: - ./keystore:/keystore:Z entrypoint: sh command: - - /opt/run_tester_node.sh + - /usr/bin/run_tester_node.sh - /usr/bin/liteprotocoltester - RECEIVER - servicenode diff --git a/apps/liteprotocoltester/lightpush_publisher.nim b/apps/liteprotocoltester/lightpush_publisher.nim index aa56bccf18..ddb2946ca5 100644 --- a/apps/liteprotocoltester/lightpush_publisher.nim +++ b/apps/liteprotocoltester/lightpush_publisher.nim @@ -17,7 +17,8 @@ import common/utils/parse_size_units, ], ./tester_config, - ./tester_message + ./tester_message, + ./lpt_metrics randomize() @@ -141,12 +142,15 @@ proc publishMessages( pubsubTopic = lightpushPubsubTopic, hash = msgHash inc(messagesSent) + lpt_publisher_sent_messages_count.inc() + lpt_publisher_sent_bytes.inc(amount = msgSize.int64) else: sentMessages[messagesSent] = (hash: msgHash, relayed: false) failedToSendCause.mgetOrPut(wlpRes.error, 1).inc() error "failed to publish message using lightpush", err = wlpRes.error, hash = msgHash inc(failedToSendCount) + lpt_publisher_failed_messages_count.inc(labelValues = [wlpRes.error]) await sleepAsync(delayMessages) diff --git a/apps/liteprotocoltester/liteprotocoltester.nim b/apps/liteprotocoltester/liteprotocoltester.nim index d7285e2f2e..3a32c04d5b 100644 --- a/apps/liteprotocoltester/liteprotocoltester.nim +++ b/apps/liteprotocoltester/liteprotocoltester.nim @@ -96,6 +96,10 @@ when isMainModule: wakuConf.clusterId = conf.clusterId ## TODO: Depending on the tester needs we might extend here with shards, clusterId, etc... + wakuConf.metricsServer = true + wakuConf.metricsServerAddress = parseIpAddress("0.0.0.0") + wakuConf.metricsServerPort = 8003 + if conf.testFunc == TesterFunctionality.SENDER: wakuConf.lightpushnode = conf.serviceNode else: @@ -108,9 +112,6 @@ when isMainModule: wakuConf.rest = false - wakuConf.metricsServer = true - wakuConf.metricsServerAddress = parseIpAddress("0.0.0.0") - # NOTE: {.threadvar.} is used to make the global variable GC safe for the closure uses it # It will always be called from main thread anyway. # Ref: https://nim-lang.org/docs/manual.html#threads-gc-safety diff --git a/apps/liteprotocoltester/lpt_metrics.nim b/apps/liteprotocoltester/lpt_metrics.nim new file mode 100644 index 0000000000..655ec098c9 --- /dev/null +++ b/apps/liteprotocoltester/lpt_metrics.nim @@ -0,0 +1,30 @@ +## Example showing how a resource restricted client may +## subscribe to messages without relay + +import metrics + +export metrics + +declarePublicGauge lpt_receiver_sender_peer_count, "count of sender peers" + +declarePublicCounter lpt_receiver_received_messages_count, + "number of messages received per peer", ["peer"] + +declarePublicCounter lpt_receiver_received_bytes, + "number of received bytes per peer", ["peer"] + +declarePublicGauge lpt_receiver_missing_messages_count, + "number of missing messages per peer", ["peer"] + +declarePublicCounter lpt_receiver_duplicate_messages_count, + "number of duplicate messages per peer", ["peer"] + +declarePublicGauge lpt_receiver_distinct_duplicate_messages_count, + "number of distinct duplicate messages per peer", ["peer"] + +declarePublicCounter lpt_publisher_sent_messages_count, "number of messages published" + +declarePublicCounter lpt_publisher_failed_messages_count, + "number of messages failed to publish per failure cause", ["cause"] + +declarePublicCounter lpt_publisher_sent_bytes, "number of total bytes sent" diff --git a/apps/liteprotocoltester/monitoring/configuration/dashboards/liter-protocol-test-monitoring.json b/apps/liteprotocoltester/monitoring/configuration/dashboards/liter-protocol-test-monitoring.json new file mode 100644 index 0000000000..96d574cfe0 --- /dev/null +++ b/apps/liteprotocoltester/monitoring/configuration/dashboards/liter-protocol-test-monitoring.json @@ -0,0 +1,1091 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitoring of lite-protocol-tester's send/receiver performance and failure counters.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "title": "Test publisher monitor", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "lpt_receiver_sender_peer_count{instance=\"receivernode:8003\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of publisher peers", + "type": "gauge" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 11, + "panels": [], + "title": "Test performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Publishing rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.gradientMode", + "value": "hue" + }, + { + "id": "custom.fillOpacity", + "value": 15 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (lpt_publisher_sent_messages_count_total)", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total published count", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (rate(lpt_publisher_sent_messages_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Publishing rate", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Publishes test messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Received message rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.scaleDistribution", + "value": { + "type": "linear" + } + }, + { + "id": "custom.fillOpacity", + "value": 9 + }, + { + "id": "custom.gradientMode", + "value": "hue" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(instance) (lpt_receiver_received_messages_count_total{instance=\"receivernode:8003\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total message received", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(instance) (rate(lpt_receiver_received_messages_count_total{instance=\"receivernode:8003\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Received message rate", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Received test messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "kbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Test message transfer rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.fillOpacity", + "value": 22 + }, + { + "id": "custom.scaleDistribution", + "value": { + "type": "linear" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (lpt_publisher_sent_bytes_total)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total received bytes", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (rate(lpt_publisher_sent_bytes_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Test message transfer rate", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Sent bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "kbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Test message transfer rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.fillOpacity", + "value": 22 + }, + { + "id": "custom.scaleDistribution", + "value": { + "type": "linear" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(instance) (lpt_receiver_received_bytes_total{instance=\"receivernode:8003\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total received bytes", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(instance) (rate(lpt_receiver_received_bytes_total{instance=\"receivernode:8003\"}[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Test message transfer rate", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Received bytes", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 10, + "panels": [], + "title": "Failure statistics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Published message rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.gradientMode", + "value": "hue" + }, + { + "id": "custom.fillOpacity", + "value": 15 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "lpt_publisher_failed_messages_count_total{instance=\"publishernode:8003\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Failed to publish count", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Failed publish count per cause", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance) (lpt_receiver_duplicate_messages_count_total{instance=\"receivernode:8003\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total duplicates", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance) (lpt_receiver_distinct_duplicate_messages_count{instance=\"receivernode:8003\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Distinct duplicates", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Received duplicated messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 21, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "lpt_receiver_missing_messages_count", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Publisher {{peer}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Not arrived messages", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Liteprotocoltester monitoring", + "uid": "fdw6pgh9odszkd", + "version": 1, + "weekStart": "" +} diff --git a/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json b/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json deleted file mode 100644 index f269b0aeb8..0000000000 --- a/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json +++ /dev/null @@ -1,5776 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 12485, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 45, - "panels": [], - "title": "Waku Node", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 0, - "y": 1 - }, - "id": 41, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(waku_histogram_message_size_bucket[1h])/scalar(rate(waku_histogram_message_size_count[1h]))*100", - "format": "heatmap", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Message distrubution %/kBytes (Last Hour)", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "deckbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 9, - "x": 9, - "y": 1 - }, - "id": 38, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_histogram_message_size_sum[1h])/rate(waku_histogram_message_size_count[1h])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Average Msg Size (Last Hour)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "deckbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 5, - "x": 9, - "y": 5 - }, - "id": 42, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, rate(waku_histogram_message_size_bucket[1h]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "75% Percentile (Last hour)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "deckbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 14, - "y": 5 - }, - "id": 39, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(waku_histogram_message_size_bucket[1h]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "99% Percentile (Last Hour)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 9 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "(increase(waku_node_messages_total[1m]))/60", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Messages/second", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "deckbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 9, - "y": 9 - }, - "id": 43, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.4.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "waku_histogram_message_size_sum/waku_histogram_message_size_count", - "format": "heatmap", - "instant": false, - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Average msg size (kBytes)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 0, - "y": 16 - }, - "id": 147, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_relay_network_bytes_total{direction=\"in\"}[$__rate_interval])", - "legendFormat": "{{topic}}", - "range": true, - "refId": "A" - } - ], - "title": "Relay traffic per shard (in)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 9, - "y": 16 - }, - "id": 148, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_relay_network_bytes_total{direction=\"out\"}[$__rate_interval])", - "legendFormat": "{{topic}}", - "range": true, - "refId": "A" - } - ], - "title": "Relay traffic per shard (out)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 25 - }, - "id": 2, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "waku_version{instance=\"nwaku:8003\"}", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Version", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "version" - ] - } - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 25 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "libp2p_autonat_reachability_confidence", - "legendFormat": "{{reachability}}", - "range": true, - "refId": "A" - } - ], - "title": "Reachability", - "transformations": [ - { - "id": "reduce", - "options": { - "includeTimeField": false, - "mode": "reduceFields", - "reducers": [ - "max" - ] - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 25 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "routing_table_nodes{state=\"seen\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Discv5 (Seen Nodes)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 25 - }, - "id": 33, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "routing_table_nodes", - "legendFormat": "{{label_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Discv5 (Nodes)", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "Time", - "{__name__=\"routing_table_nodes\", instance=\"nwaku:8003\", job=\"nwaku\"}" - ] - } - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 25 - }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "libp2p_peers", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Connected Peers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 25 - }, - "id": 28, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "libp2p_pubsub_topics", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Number Pubsub Topics", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 30 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "process_start_time_seconds{job=\"nwaku\"}*1000", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Start Times (UTC)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 15, - "x": 3, - "y": 30 - }, - "id": 44, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "waku_connected_peers", - "legendFormat": "{{direction}}_{{protocol}}", - "range": true, - "refId": "A" - } - ], - "title": "Connected Peers (Direction/Protocol)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 35 - }, - "id": 36, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "waku_peer_store_size", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Peer Store Size", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(libp2p_network_bytes_total{direction=\"in\"}[$__rate_interval])", - "legendFormat": "traffic_{{direction}}", - "range": true, - "refId": "A" - } - ], - "title": "libp2p traffic (in)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 9, - "y": 40 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(libp2p_network_bytes_total{direction=\"out\"}[$__rate_interval])", - "legendFormat": "traffic_{{direction}}", - "range": true, - "refId": "A" - } - ], - "title": "libp2p traffic (out)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 0, - "y": 49 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "libp2p_peers", - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "Connected Peers", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 6, - "y": 49 - }, - "id": 149, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_requests_total[$__rate_interval])", - "legendFormat": "{{proto}}/{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Protocol request rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 0, - "y": 59 - }, - "id": 150, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{direction=\"in\"}[$__rate_interval])", - "legendFormat": "{{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Non relay protocol traffic (in)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 9, - "y": 59 - }, - "id": 151, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{direction=\"out\"}[$__rate_interval])", - "legendFormat": "{{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Non relay protocol traffic (out)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 68 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "nim_gc_heap_instance_occupied_bytes{}", - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "Heap allocation", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 68 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "nim_gc_mem_bytes{}", - "hide": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A" - } - ], - "title": "Nim Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 68 - }, - "id": 135, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_membership_insertion_duration_seconds", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Membership Insertion (seconds)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 74 - }, - "id": 128, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_number_registered_memberships", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Registered Memberships", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 76 - }, - "id": 127, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_proof_generation_duration_seconds", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Proof Generation (seconds)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 76 - }, - "id": 126, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_proof_verification_duration_seconds", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Proof Verification (seconds)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 82 - }, - "id": 134, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_membership_credentials_import_duration_seconds", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Credentials Import (seconds)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 82 - }, - "id": 137, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_messages_total_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Messages Total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 82 - }, - "id": 136, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_proof_verification_total_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Proof Verification Total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 88 - }, - "id": 133, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_invalid_messages_total_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Invalid Messages Total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 88 - }, - "id": 130, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_spam_messages_total_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{__name__}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Spam Messages Total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 88 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "waku_rln_invalid_messages_total_total", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{type}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "RLN Invalid Messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Number of messages currently stored in the database", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 0, - "y": 94 - }, - "id": 141, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "pg_tb_stats_messages{}", - "instant": false, - "legendFormat": "{{ pubsubtopic }}", - "range": true, - "refId": "A" - } - ], - "title": "# messages per shard", - "type": "timeseries" - }, - { - "datasource": { - "type": "postgres", - "uid": "e5d2e0c2-371d-4178-ac71-edc122fb459c" - }, - "description": "Messages in local database per app name, as extracted from the content topic.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [ - { - "options": { - "/waku/2/rs/1/0": { - "index": 0, - "text": "0" - }, - "/waku/2/rs/1/1": { - "index": 1, - "text": "1" - }, - "/waku/2/rs/1/2": { - "index": 2, - "text": "2" - }, - "/waku/2/rs/1/3": { - "index": 3, - "text": "3" - }, - "/waku/2/rs/1/4": { - "index": 4, - "text": "4" - }, - "/waku/2/rs/1/5": { - "index": 5, - "text": "5" - }, - "/waku/2/rs/1/6": { - "index": 6, - "text": "6" - }, - "/waku/2/rs/1/7": { - "index": 7, - "text": "7" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Number of Messages (sum)" - }, - "properties": [ - { - "id": "unit", - "value": "none" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Total Payload Size (sum)" - }, - "properties": [ - { - "id": "unit", - "value": "decbytes" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 9, - "x": 9, - "y": 94 - }, - "id": 144, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 1, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "e5d2e0c2-371d-4178-ac71-edc122fb459c" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT REGEXP_REPLACE(contenttopic,'^\\/(.+)\\/(\\d+)\\/(.+)\\/(.+)$','\\1') as \"App name\", COUNT(id), pg_column_size(payload)\nFROM messages\nGROUP BY contenttopic, payload", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "pubsubtopic", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "name": "pubsubtopic", - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "messages" - } - ], - "title": "Stored Message by Content Topic App Name", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "contenttopic": "App name", - "count": "Number of Messages", - "pg_column_size": "Total Payload Size" - } - } - }, - { - "id": "groupBy", - "options": { - "fields": { - "App name": { - "aggregations": [ - "uniqueValues" - ], - "operation": "groupby" - }, - "Number of Messages": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "Total Payload Size": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - }, - "pg_column_size": { - "aggregations": [ - "sum" - ], - "operation": "aggregate" - } - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": true, - "field": "Number of Messages (sum)" - } - ] - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "Number of messages currently stored in the database", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 105 - }, - "id": 146, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "exemplar": false, - "expr": "pg_tb_messages_count{}", - "instant": false, - "interval": "", - "legendFormat": "messages", - "range": true, - "refId": "A" - } - ], - "title": "Unique stored messages (Postgres)", - "type": "timeseries" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Clients executing Statements.\n\nSource: pg_stat_activity", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 9, - "y": 105 - }, - "id": 23, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(pg_stat_activity_count{state=\"active\",instance=\"$Instance\"})", - "refId": "A" - } - ], - "thresholds": "", - "title": "Active clients (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 13, - "y": 105 - }, - "id": 125, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "pg_postmaster_start_time_seconds*1000", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Postgres start time", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 112 - }, - "id": 46, - "panels": [], - "title": "Postgres", - "type": "row" - }, - { - "colorBackground": false, - "colorValue": false, - "datasource": "Prometheus", - "description": "Source: server_version_num", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 113 - }, - "id": 11, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(50, 168, 82)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(pg_settings_server_version_num)", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "PostgreSQL Version", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Transactions committed + roolback per minute\n\nSource: pg_stat_database,xact_commit + xact_rollback", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 113 - }, - "id": 14, - "interval": "", - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum((rate(pg_stat_database_xact_commit{instance=\"$Instance\"}[$Interval])))+sum((rate(pg_stat_database_xact_rollback{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Transaction rate (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Statements executed per Minute.\n\nSource: pg_stat_statements.calls", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 113 - }, - "id": 93, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum((rate(pg_stat_statements_calls{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Query rate (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Source: pg_stat_statements.total_time / pg_stat_statements.calls", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 113 - }, - "id": 102, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum((delta(pg_stat_statements_total_time_seconds{instance=\"$Instance\"}[$Interval])))/sum((delta(pg_stat_statements_calls{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Average query runtime (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "decimals": 2, - "description": "Size of all databases in $Instance.\n\nSource: pg_database_size()", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 113 - }, - "id": 37, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(pg_database_size_bytes{instance=\"$Instance\"})", - "refId": "A" - } - ], - "thresholds": "", - "title": "Total database size (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Max Replication lag behind master in seconds\n\nOnly available on a standby system.\n\nSource: pg_last_xact_replay_timestamp\n\nUse: pg_stat_replication for Details.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "s", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 113 - }, - "id": 84, - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(pg_replication_lag{instance=\"$Instance\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Max Replication Lag (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "max" - }, - { - "datasource": "Prometheus", - "description": "Shared buffer hits vs reads from disc", - "fieldConfig": { - "defaults": { - "decimals": 2, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-red" - }, - { - "color": "semi-dark-yellow", - "value": 80 - }, - { - "color": "semi-dark-green", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 0, - "y": 116 - }, - "id": 16, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "expr": "sum(pg_stat_database_blks_hit{instance=~\"$Instance\"})/(sum(pg_stat_database_blks_hit{instance=~\"$Instance\"})+sum(pg_stat_database_blks_read{instance=~\"$Instance\"}))*100", - "refId": "A" - } - ], - "title": "Shared Buffer Hits (Postgres)", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "description": "Percentage of max_connections used", - "fieldConfig": { - "defaults": { - "decimals": 0, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-green" - }, - { - "color": "semi-dark-yellow", - "value": 0.75 - }, - { - "color": "semi-dark-red", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 3, - "y": 116 - }, - "id": 9, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "expr": "sum(pg_stat_database_numbackends)/max(pg_settings_max_connections)", - "refId": "A" - } - ], - "title": "Connections used (Postgres)", - "type": "gauge" - }, - { - "datasource": "Prometheus", - "description": "Transaction committed vs rollbacked", - "fieldConfig": { - "defaults": { - "decimals": 2, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-red" - }, - { - "color": "#EAB839", - "value": 0.75 - }, - { - "color": "semi-dark-green", - "value": 0.9 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 6, - "y": 116 - }, - "id": 15, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.4.2", - "targets": [ - { - "expr": "sum(pg_stat_database_xact_commit{instance=\"$Instance\"})/(sum(pg_stat_database_xact_commit{instance=\"$Instance\"}) + sum(pg_stat_database_xact_rollback{instance=\"$Instance\"}))", - "refId": "A" - } - ], - "title": "Commit Ratio (Postgres)", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 51, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 6, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 116 - }, - "id": 142, - "options": { - "legend": { - "calcs": [ - "last", - "max" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "pg_stat_user_tables_n_live_tup{datname=\"postgres\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Live", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "pg_stat_user_tables_n_dead_tup", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Dead", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Estimated number of rows (Postgres)", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "description": "View: pg_stat_activity", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 123 - }, - "hiddenSeries": false, - "id": 24, - "interval": "$Interval", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [ - { - "targetBlank": true, - "title": "PostgreSQL Documentation", - "url": "https://www.postgresql.org/docs/current/monitoring-stats.html" - } - ], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (state) (pg_stat_activity_count{instance=\"$Instance\"})", - "legendFormat": "{{state}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Connections by state (stacked) (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 0, - "description": "View: pg_stat_activity", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 124 - }, - "hiddenSeries": false, - "id": 121, - "interval": "$Interval", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [ - { - "targetBlank": true, - "title": "PostgreSQL Documentation", - "url": "https://www.postgresql.org/docs/current/monitoring-stats.html" - } - ], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (datname) (pg_stat_activity_count{instance=\"$Instance\"})", - "legendFormat": "{{datname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Connections by database (stacked) (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - }, - { - "decimals": 0, - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "decimals": 2, - "description": "1 Minute rate of transactions committed or rollback.", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 132 - }, - "hiddenSeries": false, - "id": 122, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum ((rate(pg_stat_database_xact_commit[$Interval])))", - "interval": "", - "legendFormat": "committed", - "refId": "A" - }, - { - "expr": "sum ((rate(pg_stat_database_xact_rollback[$Interval])))", - "hide": false, - "interval": "", - "legendFormat": "rollback", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Transactions (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Source: pg_stat_database", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 133 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum((rate(pg_stat_database_tup_inserted{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "Inserts", - "refId": "A" - }, - { - "expr": "sum((rate(pg_stat_database_tup_updated{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "Updates", - "refId": "B" - }, - { - "expr": "sum((rate(pg_stat_database_tup_deleted{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "Deletes", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tuples inserts/updates/deletes (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "* blk_read_time: Time spent reading data file blocks by backends in this database, in milliseconds\n* blk_write_time: Time spent writing data file blocks by backends in this database, in milliseconds\n\ntrack_io_timings needs to be activated", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 141 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum ((rate(pg_stat_database_blk_read_time{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "blk_read_time", - "refId": "A" - }, - { - "expr": "sum ((rate(pg_stat_database_blk_write_time{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "blk_read_time", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "I/O Read/Write time (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Source: pg_stat_database\n\n* tup_fetched: rows needed to satisfy queries\n* tup_returned: rows read/scanned", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 142 - }, - "hiddenSeries": false, - "id": 111, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideZero": false, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum((rate(pg_stat_database_tup_fetched{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "Fetched", - "refId": "A" - }, - { - "expr": "sum((rate(pg_stat_database_tup_returned{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "Returned", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Tuples fetched/returned (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Source: pg_locks", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 150 - }, - "hiddenSeries": false, - "id": 123, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - { - "title": "PostgreSQL Lock Modes", - "url": "https://www.postgresql.org/docs/12/explicit-locking.html#LOCKING-TABLES" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (mode) (pg_locks_count{instance=\"$Instance\"})", - "legendFormat": "{{mode}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Locks by state (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Should be 0 \n\nSource: pg_stat_database\n\nWith log_lock_waits turned on, deadlocks will be logged to the PostgreSQL Logfiles.", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 151 - }, - "hiddenSeries": false, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - { - "title": "PostgreSQL Locking", - "url": "https://www.postgresql.org/docs/12/explicit-locking.html" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (datname) ((rate(pg_stat_database_deadlocks{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "{{datname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Deadlocks by database (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Should be 0. If temporary files are created, it can indicate insufficient work_mem. With log_temp_files the creation of temporary files are logged to the PostgreSQL Logfiles.", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 156 - }, - "hiddenSeries": false, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - { - "title": "PostgreSQL Ressources", - "url": "https://www.postgresql.org/docs/current/runtime-config-resource.html" - } - ], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (datname) ((rate(pg_stat_database_temp_files{instance=\"$Instance\"}[$Interval])))", - "interval": "", - "legendFormat": "{{datname}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Temporary files by database (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Lag behind master in seconds.\n\nOnly available on a standby System.", - "fieldConfig": { - "defaults": { - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 160 - }, - "hiddenSeries": false, - "id": 120, - "interval": "1m", - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(pg_replication_lag{instance=\"$Instance\"})", - "instant": false, - "intervalFactor": 1, - "legendFormat": "lag ", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Replication lag (Postgres)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "revision": 1, - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values({job=\"postgres-exporter\"}, instance)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "Instance", - "options": [], - "query": "label_values({job=\"postgres-exporter\"}, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(datname)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "Database", - "options": [], - "query": "label_values(datname)", - "refresh": 1, - "regex": "/^(?!template*|postgres).*$/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "10m", - "value": "10m" - }, - "hide": 0, - "name": "Interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_Interval" - }, - { - "selected": false, - "text": "30sec", - "value": "30sec" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": true, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - } - ], - "query": "30sec,1m,10m,30m,1h,6h,12h,1d", - "queryValue": "", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "2024-06-26T02:42:06.763Z", - "to": "2024-06-26T04:42:06.771Z" - }, - "timepicker": { - "refresh_intervals": [ - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "browser", - "title": "nwaku-monitoring", - "uid": "yns_4vFVk", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/apps/liteprotocoltester/monitoring/configuration/datasources.yaml b/apps/liteprotocoltester/monitoring/configuration/datasources.yaml index 9f4f51f86a..2cc211feeb 100644 --- a/apps/liteprotocoltester/monitoring/configuration/datasources.yaml +++ b/apps/liteprotocoltester/monitoring/configuration/datasources.yaml @@ -5,7 +5,7 @@ datasources: type: prometheus access: proxy org_id: 1 - url: http://prometheus:9090 + url: http://prometheus:9099 is_default: true version: 1 - editable: true \ No newline at end of file + editable: true diff --git a/apps/liteprotocoltester/monitoring/configuration/grafana.ini b/apps/liteprotocoltester/monitoring/configuration/grafana.ini index f237726b3e..631fbb73af 100644 --- a/apps/liteprotocoltester/monitoring/configuration/grafana.ini +++ b/apps/liteprotocoltester/monitoring/configuration/grafana.ini @@ -1,9 +1,11 @@ -instance_name = nwaku dashboard +instance_name = liteprotocoltester dashboard ;[dashboards.json] ;enabled = true ;path = /home/git/grafana/grafana-dashboards/dashboards +[server] +http_port = 3033 #################################### Auth ########################## [auth] diff --git a/apps/liteprotocoltester/monitoring/configuration/pg-exporter-queries.yml b/apps/liteprotocoltester/monitoring/configuration/pg-exporter-queries.yml deleted file mode 100644 index bb1d7320a1..0000000000 --- a/apps/liteprotocoltester/monitoring/configuration/pg-exporter-queries.yml +++ /dev/null @@ -1,284 +0,0 @@ -pg_replication: - query: "SELECT CASE WHEN NOT pg_is_in_recovery() THEN 0 ELSE GREATEST (0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) END AS lag" - master: true - metrics: - - lag: - usage: "GAUGE" - description: "Replication lag behind master in seconds" - -pg_postmaster: - query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" - master: true - metrics: - - start_time_seconds: - usage: "GAUGE" - description: "Time at which postmaster started" - -pg_stat_user_tables: - query: | - SELECT - current_database() datname, - schemaname, - relname, - seq_scan, - seq_tup_read, - idx_scan, - idx_tup_fetch, - n_tup_ins, - n_tup_upd, - n_tup_del, - n_tup_hot_upd, - n_live_tup, - n_dead_tup, - n_mod_since_analyze, - COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, - COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, - COALESCE(last_analyze, '1970-01-01Z') as last_analyze, - COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, - vacuum_count, - autovacuum_count, - analyze_count, - autoanalyze_count - FROM - pg_stat_user_tables - metrics: - - datname: - usage: "LABEL" - description: "Name of current database" - - schemaname: - usage: "LABEL" - description: "Name of the schema that this table is in" - - relname: - usage: "LABEL" - description: "Name of this table" - - seq_scan: - usage: "COUNTER" - description: "Number of sequential scans initiated on this table" - - seq_tup_read: - usage: "COUNTER" - description: "Number of live rows fetched by sequential scans" - - idx_scan: - usage: "COUNTER" - description: "Number of index scans initiated on this table" - - idx_tup_fetch: - usage: "COUNTER" - description: "Number of live rows fetched by index scans" - - n_tup_ins: - usage: "COUNTER" - description: "Number of rows inserted" - - n_tup_upd: - usage: "COUNTER" - description: "Number of rows updated" - - n_tup_del: - usage: "COUNTER" - description: "Number of rows deleted" - - n_tup_hot_upd: - usage: "COUNTER" - description: "Number of rows HOT updated (i.e., with no separate index update required)" - - n_live_tup: - usage: "GAUGE" - description: "Estimated number of live rows" - - n_dead_tup: - usage: "GAUGE" - description: "Estimated number of dead rows" - - n_mod_since_analyze: - usage: "GAUGE" - description: "Estimated number of rows changed since last analyze" - - last_vacuum: - usage: "GAUGE" - description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" - - last_autovacuum: - usage: "GAUGE" - description: "Last time at which this table was vacuumed by the autovacuum daemon" - - last_analyze: - usage: "GAUGE" - description: "Last time at which this table was manually analyzed" - - last_autoanalyze: - usage: "GAUGE" - description: "Last time at which this table was analyzed by the autovacuum daemon" - - vacuum_count: - usage: "COUNTER" - description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" - - autovacuum_count: - usage: "COUNTER" - description: "Number of times this table has been vacuumed by the autovacuum daemon" - - analyze_count: - usage: "COUNTER" - description: "Number of times this table has been manually analyzed" - - autoanalyze_count: - usage: "COUNTER" - description: "Number of times this table has been analyzed by the autovacuum daemon" - -pg_statio_user_tables: - query: "SELECT current_database() datname, schemaname, relname, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables" - metrics: - - datname: - usage: "LABEL" - description: "Name of current database" - - schemaname: - usage: "LABEL" - description: "Name of the schema that this table is in" - - relname: - usage: "LABEL" - description: "Name of this table" - - heap_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table" - - heap_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table" - - idx_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from all indexes on this table" - - idx_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in all indexes on this table" - - toast_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table's TOAST table (if any)" - - toast_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table's TOAST table (if any)" - - tidx_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table's TOAST table indexes (if any)" - - tidx_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table's TOAST table indexes (if any)" - -# WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series -pg_stat_statements: - query: "SELECT t2.rolname, t3.datname, queryid, calls, ( total_plan_time + total_exec_time ) / 1000 as total_time_seconds, ( min_plan_time + min_exec_time ) / 1000 as min_time_seconds, ( max_plan_time + max_exec_time ) / 1000 as max_time_seconds, ( mean_plan_time + mean_exec_time ) / 1000 as mean_time_seconds, ( stddev_plan_time + stddev_exec_time ) / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin' AND queryid IS NOT NULL" - master: true - metrics: - - rolname: - usage: "LABEL" - description: "Name of user" - - datname: - usage: "LABEL" - description: "Name of database" - - queryid: - usage: "LABEL" - description: "Query ID" - - calls: - usage: "COUNTER" - description: "Number of times executed" - - total_time_seconds: - usage: "COUNTER" - description: "Total time spent in the statement, in milliseconds" - - min_time_seconds: - usage: "GAUGE" - description: "Minimum time spent in the statement, in milliseconds" - - max_time_seconds: - usage: "GAUGE" - description: "Maximum time spent in the statement, in milliseconds" - - mean_time_seconds: - usage: "GAUGE" - description: "Mean time spent in the statement, in milliseconds" - - stddev_time_seconds: - usage: "GAUGE" - description: "Population standard deviation of time spent in the statement, in milliseconds" - - rows: - usage: "COUNTER" - description: "Total number of rows retrieved or affected by the statement" - - shared_blks_hit: - usage: "COUNTER" - description: "Total number of shared block cache hits by the statement" - - shared_blks_read: - usage: "COUNTER" - description: "Total number of shared blocks read by the statement" - - shared_blks_dirtied: - usage: "COUNTER" - description: "Total number of shared blocks dirtied by the statement" - - shared_blks_written: - usage: "COUNTER" - description: "Total number of shared blocks written by the statement" - - local_blks_hit: - usage: "COUNTER" - description: "Total number of local block cache hits by the statement" - - local_blks_read: - usage: "COUNTER" - description: "Total number of local blocks read by the statement" - - local_blks_dirtied: - usage: "COUNTER" - description: "Total number of local blocks dirtied by the statement" - - local_blks_written: - usage: "COUNTER" - description: "Total number of local blocks written by the statement" - - temp_blks_read: - usage: "COUNTER" - description: "Total number of temp blocks read by the statement" - - temp_blks_written: - usage: "COUNTER" - description: "Total number of temp blocks written by the statement" - - blk_read_time_seconds: - usage: "COUNTER" - description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" - - blk_write_time_seconds: - usage: "COUNTER" - description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" - -pg_process_idle: - query: | - WITH - metrics AS ( - SELECT - application_name, - SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum, - COUNT(*) AS process_idle_seconds_count - FROM pg_stat_activity - WHERE state = 'idle' - GROUP BY application_name - ), - buckets AS ( - SELECT - application_name, - le, - SUM( - CASE WHEN EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change)) <= le - THEN 1 - ELSE 0 - END - )::bigint AS bucket - FROM - pg_stat_activity, - UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le - GROUP BY application_name, le - ORDER BY application_name, le - ) - SELECT - application_name, - process_idle_seconds_sum as seconds_sum, - process_idle_seconds_count as seconds_count, - ARRAY_AGG(le) AS seconds, - ARRAY_AGG(bucket) AS seconds_bucket - FROM metrics JOIN buckets USING (application_name) - GROUP BY 1, 2, 3 - metrics: - - application_name: - usage: "LABEL" - description: "Application Name" - - seconds: - usage: "HISTOGRAM" - description: "Idle time of server processes" - -pg_tb_stats: - query: | - select pubsubtopic, count(*) AS messages FROM (SELECT id, array_agg(pubsubtopic ORDER BY pubsubtopic) AS pubsubtopic FROM messages GROUP BY id) sub GROUP BY pubsubtopic ORDER BY pubsubtopic; - metrics: - - pubsubtopic: - usage: "LABEL" - description: "pubsubtopic" - - messages: - usage: "GAUGE" - description: "Number of messages for the given pubsub topic" - -pg_tb_messages: - query: | - SELECT - COUNT(ID) - FROM messages - metrics: - - count: - usage: "GAUGE" - description: "Row count in `messages` table" diff --git a/apps/liteprotocoltester/monitoring/configuration/postgres-exporter.yml b/apps/liteprotocoltester/monitoring/configuration/postgres-exporter.yml deleted file mode 100644 index a8380dd72e..0000000000 --- a/apps/liteprotocoltester/monitoring/configuration/postgres-exporter.yml +++ /dev/null @@ -1,9 +0,0 @@ -auth_modules: - mypostgres: - type: userpass - userpass: - username: postgres - password: ${POSTGRES_PASSWORD} - options: - # options become key=value parameters of the DSN - sslmode: disable diff --git a/apps/liteprotocoltester/monitoring/prometheus-config.yml b/apps/liteprotocoltester/monitoring/prometheus-config.yml index 650798fdcf..7ea2d1747a 100644 --- a/apps/liteprotocoltester/monitoring/prometheus-config.yml +++ b/apps/liteprotocoltester/monitoring/prometheus-config.yml @@ -5,6 +5,14 @@ global: monitor: "Monitoring" scrape_configs: - - job_name: "nwaku" + - job_name: "liteprotocoltester" static_configs: - - targets: ["nwaku:8003"] + - targets: ["lightpush-service:8003", + "filter-service:8003", + "liteprotocoltester-publishernode-1:8003", + "liteprotocoltester-publishernode-2:8003", + "liteprotocoltester-publishernode-3:8003", + "liteprotocoltester-publishernode-4:8003", + "liteprotocoltester-publishernode-5:8003", + "liteprotocoltester-publishernode-6:8003", + "receivernode:8003"] diff --git a/apps/liteprotocoltester/run_service_node.sh b/apps/liteprotocoltester/run_service_node.sh index fdbbf6cdf3..01ad00e00e 100755 --- a/apps/liteprotocoltester/run_service_node.sh +++ b/apps/liteprotocoltester/run_service_node.sh @@ -56,6 +56,7 @@ exec /usr/bin/wakunode\ --discv5-bootstrap-node=${BOOTSTRAP_ENR}\ --log-level=INFO\ --metrics-server=True\ + --metrics-server-port=8003\ --metrics-server-address=0.0.0.0\ --nat=extip:${IP}\ ${PUBSUB}\ diff --git a/apps/liteprotocoltester/run_tester_node.sh b/apps/liteprotocoltester/run_tester_node.sh index c5a5355c6d..c1e77cc3df 100755 --- a/apps/liteprotocoltester/run_tester_node.sh +++ b/apps/liteprotocoltester/run_tester_node.sh @@ -7,7 +7,6 @@ if test -f .env; then . $(pwd)/.env fi -IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') echo "I am a lite-protocol-tester node" @@ -52,8 +51,12 @@ fi if [ "${SERIVCE_NODE_ADDR}" = "waku-sim" ]; then DO_DETECT_SERVICENODE=1 SERIVCE_NODE_ADDR="" + MY_EXT_IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/') +else + MY_EXT_IP=$(wget -qO- --no-check-certificate https://api4.ipify.org) fi + if [ $DO_DETECT_SERVICENODE -eq 1 ]; then RETRIES=${RETRIES:=10} @@ -112,10 +115,12 @@ fi echo "Running binary: ${BINARY_PATH}" echo "Tester node: ${FUNCTION}" echo "Using service node: ${SERIVCE_NODE_ADDR}" +echo "My external IP: ${MY_EXT_IP}" exec "${BINARY_PATH}"\ --log-level=INFO\ --service-node="${SERIVCE_NODE_ADDR}"\ + --nat=extip:${MY_EXT_IP}\ ${DELAY_MESSAGES}\ ${NUM_MESSAGES}\ ${PUBSUB}\ @@ -125,5 +130,4 @@ exec "${BINARY_PATH}"\ ${START_PUBLISHING_AFTER}\ ${MIN_MESSAGE_SIZE}\ ${MAX_MESSAGE_SIZE} - # --nat=extip:${IP}\ # --config-file=config.toml\ diff --git a/apps/liteprotocoltester/statistics.nim b/apps/liteprotocoltester/statistics.nim index 1397266c53..003d981663 100644 --- a/apps/liteprotocoltester/statistics.nim +++ b/apps/liteprotocoltester/statistics.nim @@ -5,9 +5,10 @@ import chronos/timer as chtimer, chronicles, chronos, - results + results, + libp2p/peerid -import ./tester_message +import ./tester_message, ./lpt_metrics type ArrivalInfo = object @@ -54,7 +55,9 @@ proc init*(T: type Statistics, expectedMessageCount: int = 1000): T = result.received = initTable[uint32, MessageInfo](expectedMessageCount) return result -proc addMessage*(self: var Statistics, msg: ProtocolTesterMessage, msgHash: string) = +proc addMessage*( + self: var Statistics, sender: string, msg: ProtocolTesterMessage, msgHash: string +) = if self.allMessageCount == 0: self.allMessageCount = msg.count self.firstReceivedIdx = msg.index @@ -70,7 +73,7 @@ proc addMessage*(self: var Statistics, msg: ProtocolTesterMessage, msgHash: stri prevIndex: self.helper.prevIndex, ), ) - + lpt_receiver_received_bytes.inc(labelValues = [sender], amount = msg.size.int64) if self.received.hasKeyOrPut(msg.index, currentArrived): inc(self.duplicateCount) self.helper.duplicates.mgetOrPut(msg.index, (msgHash, 0, msg.size)).dupCount.inc() @@ -78,6 +81,10 @@ proc addMessage*(self: var Statistics, msg: ProtocolTesterMessage, msgHash: stri index = msg.index, hash = msgHash, times_duplicated = self.helper.duplicates[msg.index].dupCount + lpt_receiver_duplicate_messages_count.inc(labelValues = [sender]) + lpt_receiver_distinct_duplicate_messages_count.set( + labelValues = [sender], value = self.helper.duplicates.len() + ) return ## detect misorder arrival and possible lost messages @@ -93,6 +100,10 @@ proc addMessage*(self: var Statistics, msg: ProtocolTesterMessage, msgHash: stri self.helper.prevIndex = msg.index self.helper.prevArrivedAt = currentArrived.info.arrivedAt inc(self.receivedMessages) + lpt_receiver_received_messages_count.inc(labelValues = [sender]) + lpt_receiver_missing_messages_count.set( + labelValues = [sender], value = (self.helper.maxIndex - self.receivedMessages).int64 + ) proc addMessage*( self: var PerPeerStatistics, @@ -103,8 +114,17 @@ proc addMessage*( if not self.contains(peerId): self[peerId] = Statistics.init() + let shortSenderId = block: + let senderPeer = PeerId.init(msg.sender) + if senderPeer.isErr: + msg.sender + else: + senderPeer.get().shortLog() + discard catch: - self[peerId].addMessage(msg, msgHash) + self[peerId].addMessage(shortSenderId, msg, msgHash) + + lpt_receiver_sender_peer_count.set(value = self.len) proc lossCount*(self: Statistics): uint32 = self.helper.maxIndex - self.receivedMessages From 2ca0e6c3a45d2368ae9ac5c5d78e9d17e709424a Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:40:49 +0200 Subject: [PATCH 2/3] Update apps/liteprotocoltester/statistics.nim Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> --- apps/liteprotocoltester/statistics.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/liteprotocoltester/statistics.nim b/apps/liteprotocoltester/statistics.nim index 003d981663..db4a8a81a1 100644 --- a/apps/liteprotocoltester/statistics.nim +++ b/apps/liteprotocoltester/statistics.nim @@ -116,7 +116,7 @@ proc addMessage*( let shortSenderId = block: let senderPeer = PeerId.init(msg.sender) - if senderPeer.isErr: + if senderPeer.isErr(): msg.sender else: senderPeer.get().shortLog() From 7dad8833f2ac6ae9b84dfca4446826280480179e Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Tue, 3 Sep 2024 22:09:51 +0200 Subject: [PATCH 3/3] indent fix, more stable finding of service/bootstrap nodes, pre-set for TWN --- apps/liteprotocoltester/.env | 16 ++++++++-------- apps/liteprotocoltester/liteprotocoltester.nim | 2 +- apps/liteprotocoltester/run_service_node.sh | 4 ++-- apps/liteprotocoltester/run_tester_node.sh | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/liteprotocoltester/.env b/apps/liteprotocoltester/.env index a65106e737..0d0d0ef6c4 100644 --- a/apps/liteprotocoltester/.env +++ b/apps/liteprotocoltester/.env @@ -1,10 +1,10 @@ -START_PUBLISHING_AFTER=10 +START_PUBLISHING_AFTER=45 # can add some seconds delay before SENDER starts publishing NUM_MESSAGES=0 # 0 for infinite number of messages -DELAY_MESSAGES=188 +DELAY_MESSAGES=8000 # ms delay between messages @@ -12,9 +12,9 @@ MIN_MESSAGE_SIZE=15Kb MAX_MESSAGE_SIZE=145Kb ## for wakusim -PUBSUB=/waku/2/rs/66/0 -CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim -CLUSTER_ID=66 +#PUBSUB=/waku/2/rs/66/0 +#CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim +#CLUSTER_ID=66 ## for status.prod #PUBSUB=/waku/2/rs/16/32 @@ -22,6 +22,6 @@ CLUSTER_ID=66 #CLUSTER_ID=16 ## for TWN -#PUBSUB=/waku/2/rs/1/4 -#CONTENT_TOPIC=/tester/2/light-pubsub-test/twn -#CLUSTER_ID=1 +PUBSUB=/waku/2/rs/1/4 +CONTENT_TOPIC=/tester/2/light-pubsub-test/twn +CLUSTER_ID=1 diff --git a/apps/liteprotocoltester/liteprotocoltester.nim b/apps/liteprotocoltester/liteprotocoltester.nim index 3a32c04d5b..a4589b60b6 100644 --- a/apps/liteprotocoltester/liteprotocoltester.nim +++ b/apps/liteprotocoltester/liteprotocoltester.nim @@ -98,7 +98,7 @@ when isMainModule: wakuConf.metricsServer = true wakuConf.metricsServerAddress = parseIpAddress("0.0.0.0") - wakuConf.metricsServerPort = 8003 + wakuConf.metricsServerPort = 8003 if conf.testFunc == TesterFunctionality.SENDER: wakuConf.lightpushnode = conf.serviceNode diff --git a/apps/liteprotocoltester/run_service_node.sh b/apps/liteprotocoltester/run_service_node.sh index 01ad00e00e..1d36292c11 100755 --- a/apps/liteprotocoltester/run_service_node.sh +++ b/apps/liteprotocoltester/run_service_node.sh @@ -19,12 +19,12 @@ echo "STANDALONE: ${STANDALONE}" if [ -z "${STANDALONE}" ]; then - RETRIES=${RETRIES:=10} + RETRIES=${RETRIES:=20} while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/'); echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})" - sleep 1 + sleep 3 RETRIES=$(( $RETRIES - 1 )) done diff --git a/apps/liteprotocoltester/run_tester_node.sh b/apps/liteprotocoltester/run_tester_node.sh index c1e77cc3df..146eb1b25d 100755 --- a/apps/liteprotocoltester/run_tester_node.sh +++ b/apps/liteprotocoltester/run_tester_node.sh @@ -58,7 +58,7 @@ fi if [ $DO_DETECT_SERVICENODE -eq 1 ]; then - RETRIES=${RETRIES:=10} + RETRIES=${RETRIES:=20} while [ -z "${SERIVCE_NODE_ADDR}" ] && [ ${RETRIES} -ge 0 ]; do SERVICE_DEBUG_INFO=$(wget -qO- http://${SERVICENAME}:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null); @@ -66,7 +66,7 @@ if [ $DO_DETECT_SERVICENODE -eq 1 ]; then SERIVCE_NODE_ADDR=$(wget -qO- http://${SERVICENAME}:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"listenAddresses":\["\([^"]*\)".*/\1/'); echo "Service node not ready, retrying (retries left: ${RETRIES})" - sleep 1 + sleep 3 RETRIES=$(( $RETRIES - 1 )) done