From e17252e9a0f1aaab3af1f46123c8ee9f77ff4d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Tue, 9 Jul 2024 10:43:33 +0200 Subject: [PATCH] [NU-1726] new quickstart (#176) --- .github/workflows/pr-newquickstart-test.yml | 36 + new-quickstart/.env | 1 + .../designer/application-customizations.conf | 16 + new-quickstart/docker-compose.yml | 269 ++ new-quickstart/download-and-start.sh | 11 + new-quickstart/flink/Dockerfile | 17 + new-quickstart/flink/flink-properties.yml | 28 + .../grafana/dashboards/dashboards.yml | 13 + .../dashboards/nussknacker-lite-scenario.json | 2320 +++++++++++ ...nussknacker-request-response-scenario.json | 2346 +++++++++++ .../dashboards/nussknacker-scenario.json | 3467 +++++++++++++++++ .../grafana/datasources/datasources.yml | 13 + new-quickstart/nginx/additional/akhq.conf | 3 + new-quickstart/nginx/additional/flink.conf | 3 + new-quickstart/nginx/nginx.conf | 32 + new-quickstart/quickstart-setup/Dockerfile | 18 + .../data/http/generate-requests/loan.sh | 12 + .../quickstart-setup/data/http/slugs.txt | 2 + .../data/http/static-requests/loan.txt | 7 + .../kafka/generate-messages/transactions.sh | 11 + .../kafka/static-messages/transactions.txt | 3 + .../quickstart-setup/data/kafka/topics.txt | 2 + new-quickstart/quickstart-setup/entrypoint.sh | 14 + .../auto-executed/001-setup-schemas.sh | 58 + .../scripts/auto-executed/002-setup-topics.sh | 30 + ...003-import-and-deploy-example-scenarios.sh | 34 + .../004-send-kafka-static-messages copy.sh | 43 + .../005-send-http-static-requests.sh | 44 + ...tinuously-send-kafka-generated-messages.sh | 38 + ...ntinuously-send-http-generated-requests.sh | 38 + .../http/continuously-send-http-requests.sh | 20 + .../send-request-to-nu-openapi-service.sh | 26 + .../utils/kafka/continuously-send-to-topic.sh | 20 + .../utils/kafka/create-topic-idempotently.sh | 12 + .../scripts/utils/kafka/purge-topic.sh | 17 + .../scripts/utils/kafka/read-from-topic.sh | 12 + .../scripts/utils/kafka/send-to-topic.sh | 13 + .../quickstart-setup/scripts/utils/lib.sh | 36 + ...loy-scenario-and-wait-for-running-state.sh | 96 + .../utils/nu/load-scenario-from-json-file.sh | 161 + .../utils/nu/load-scenario-from-json.sh | 17 + .../quickstart-setup/setup/kafka/topics.txt | 5 + .../quickstart-setup/setup/nu/examples.txt | 4 + .../nu/scenarios/DetectLargeTransactions.json | 125 + .../nu/scenarios/DetermineOfferedPlan.json | 190 + .../setup/nu/scenarios/LoanRequest.json | 239 ++ .../setup/schema-registry/active-schemas.txt | 5 + .../schemas/Customers.schema.json | 30 + .../schemas/ProcessedTransactions.schema.json | 12 + .../schemas/SmsesWithOffer.schema.json | 19 + .../schemas/Transactions.schema.json | 12 + new-quickstart/start.sh | 40 + new-quickstart/stop-and-clean.sh | 26 + new-quickstart/telegraf/telegraf.conf | 52 + 54 files changed, 10118 insertions(+) create mode 100644 .github/workflows/pr-newquickstart-test.yml create mode 100644 new-quickstart/.env create mode 100644 new-quickstart/designer/application-customizations.conf create mode 100644 new-quickstart/docker-compose.yml create mode 100755 new-quickstart/download-and-start.sh create mode 100644 new-quickstart/flink/Dockerfile create mode 100644 new-quickstart/flink/flink-properties.yml create mode 100644 new-quickstart/grafana/dashboards/dashboards.yml create mode 100644 new-quickstart/grafana/dashboards/nussknacker-lite-scenario.json create mode 100644 new-quickstart/grafana/dashboards/nussknacker-request-response-scenario.json create mode 100644 new-quickstart/grafana/dashboards/nussknacker-scenario.json create mode 100644 new-quickstart/grafana/datasources/datasources.yml create mode 100644 new-quickstart/nginx/additional/akhq.conf create mode 100644 new-quickstart/nginx/additional/flink.conf create mode 100644 new-quickstart/nginx/nginx.conf create mode 100644 new-quickstart/quickstart-setup/Dockerfile create mode 100755 new-quickstart/quickstart-setup/data/http/generate-requests/loan.sh create mode 100644 new-quickstart/quickstart-setup/data/http/slugs.txt create mode 100644 new-quickstart/quickstart-setup/data/http/static-requests/loan.txt create mode 100755 new-quickstart/quickstart-setup/data/kafka/generate-messages/transactions.sh create mode 100644 new-quickstart/quickstart-setup/data/kafka/static-messages/transactions.txt create mode 100644 new-quickstart/quickstart-setup/data/kafka/topics.txt create mode 100755 new-quickstart/quickstart-setup/entrypoint.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/001-setup-schemas.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/002-setup-topics.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/003-import-and-deploy-example-scenarios.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/004-send-kafka-static-messages copy.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/005-send-http-static-requests.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/006-continuously-send-kafka-generated-messages.sh create mode 100755 new-quickstart/quickstart-setup/scripts/auto-executed/007-continuously-send-http-generated-requests.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/http/continuously-send-http-requests.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/http/send-request-to-nu-openapi-service.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/kafka/continuously-send-to-topic.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/kafka/create-topic-idempotently.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/kafka/purge-topic.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/kafka/read-from-topic.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/kafka/send-to-topic.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/lib.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/nu/deploy-scenario-and-wait-for-running-state.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json-file.sh create mode 100755 new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json.sh create mode 100644 new-quickstart/quickstart-setup/setup/kafka/topics.txt create mode 100644 new-quickstart/quickstart-setup/setup/nu/examples.txt create mode 100644 new-quickstart/quickstart-setup/setup/nu/scenarios/DetectLargeTransactions.json create mode 100644 new-quickstart/quickstart-setup/setup/nu/scenarios/DetermineOfferedPlan.json create mode 100644 new-quickstart/quickstart-setup/setup/nu/scenarios/LoanRequest.json create mode 100644 new-quickstart/quickstart-setup/setup/schema-registry/active-schemas.txt create mode 100644 new-quickstart/quickstart-setup/setup/schema-registry/schemas/Customers.schema.json create mode 100644 new-quickstart/quickstart-setup/setup/schema-registry/schemas/ProcessedTransactions.schema.json create mode 100644 new-quickstart/quickstart-setup/setup/schema-registry/schemas/SmsesWithOffer.schema.json create mode 100644 new-quickstart/quickstart-setup/setup/schema-registry/schemas/Transactions.schema.json create mode 100755 new-quickstart/start.sh create mode 100755 new-quickstart/stop-and-clean.sh create mode 100644 new-quickstart/telegraf/telegraf.conf diff --git a/.github/workflows/pr-newquickstart-test.yml b/.github/workflows/pr-newquickstart-test.yml new file mode 100644 index 00000000..433e111a --- /dev/null +++ b/.github/workflows/pr-newquickstart-test.yml @@ -0,0 +1,36 @@ +name: QUICKSTART CI + +on: + pull_request: + branches: + - main + - staging + - release/* + push: + branches: + - main + - staging + - release/* + +jobs: + + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v3 + - name: Test Quickstart start + timeout-minutes: 10 + run: | + ./new-quickstart/start.sh || (docker-compose logs && exit 1) + - name: Clean up Quickstart + run: | + ./new-quickstart/stop-and-clean.sh + \ No newline at end of file diff --git a/new-quickstart/.env b/new-quickstart/.env new file mode 100644 index 00000000..c26fbbc2 --- /dev/null +++ b/new-quickstart/.env @@ -0,0 +1 @@ +NUSSKNACKER_VERSION=1.16.0-RC1 \ No newline at end of file diff --git a/new-quickstart/designer/application-customizations.conf b/new-quickstart/designer/application-customizations.conf new file mode 100644 index 00000000..9efe87bb --- /dev/null +++ b/new-quickstart/designer/application-customizations.conf @@ -0,0 +1,16 @@ +## To customize Nu Designer configuration see https://nussknacker.io/documentation/docs/installation_configuration_guide/Common/ + +scenarioTypes { + "streaming" { + # customize Flink streaming scenario type + # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file + } + "streaming-lite-embedded" { + # customize Lite streaming scenario type + # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file + } + "request-response-embedded": { + # customize Lite request-response scenario type + # see: https://nussknacker.io/documentation/docs/next/installation_configuration_guide/Common/#configuration-file + } +} diff --git a/new-quickstart/docker-compose.yml b/new-quickstart/docker-compose.yml new file mode 100644 index 00000000..e815c6bf --- /dev/null +++ b/new-quickstart/docker-compose.yml @@ -0,0 +1,269 @@ +name: nussknacker + +services: + + ### Quickstart setup container + quickstart-setup: + build: + context: quickstart-setup/ + dockerfile: Dockerfile + depends_on: + nginx: + condition: service_healthy + deploy: + resources: + limits: + memory: 128M + + ### Nussknacker-related services + + nginx: + image: nginx:1.26.0-alpine + restart: unless-stopped + ports: + - 8080:8080 + - 8181:8181 + depends_on: + grafana: + condition: service_healthy + designer: + condition: service_healthy + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8080 || exit 1" ] + interval: 10s + retries: 10 + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./nginx/additional:/etc/nginx/additional + deploy: + resources: + limits: + memory: 64M + + ### Nussknacker designer service + + designer: + image: touk/nussknacker:${NUSSKNACKER_VERSION:?NUSSKNACKER_VERSION must be defined}_scala-2.12 + restart: unless-stopped + environment: + CONFIG_FILE: "/opt/nussknacker/conf/application.conf,/opt/nussknacker/conf/application-customizations.conf" + DB_URL: "jdbc:postgresql://postgres:5432/nu-db" + DB_DRIVER: "org.postgresql.Driver" + DB_USER: "nu" + DB_PASSWORD: "nupassword" + NUSSKNACKER_LOG_LEVEL: "INFO" + KAFKA_ADDRESS: "kafka:9092" + SCHEMA_REGISTRY_URL: "http://schema-registry:8081" + INFLUXDB_URL: "http://influxdb:8086" + FLINK_REST_URL: "http://flink-jobmanager:8081" + JDK_JAVA_OPTIONS: "-Xmx1024M" + USAGE_REPORTS_SOURCE: "quickstart-docker-compose" + depends_on: + postgres: + condition: service_healthy + grafana: + condition: service_healthy + kafka: + condition: service_healthy + schema-registry: + condition: service_healthy + flink-jobmanager: + condition: service_healthy + flink-taskmanager: + condition: service_started + expose: + - 8181 + healthcheck: + test: [ "CMD-SHELL", "curl localhost:8080/api/app/healthCheck" ] + interval: 10s + retries: 10 + volumes: + - ./designer/application-customizations.conf:/opt/nussknacker/conf/application-customizations.conf + volumes_from: + - flink-jobmanager + deploy: + resources: + limits: + memory: 1024M + + postgres: + image: postgres:13 + restart: unless-stopped + environment: + POSTGRES_DB: "nu-db" + POSTGRES_USER: "nu" + POSTGRES_PASSWORD: "nupassword" + healthcheck: + test: ["CMD-SHELL", "pg_isready -d nu-db -U nu"] + interval: 10s + retries: 10 + volumes: + - nussknacker_designer_data:/var/lib/postgresql/data + deploy: + resources: + limits: + memory: 256M + + ### Nussknacker metrics-related services: + + grafana: + image: grafana/grafana:10.1.10 + restart: unless-stopped + environment: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_SERVER_ROOT_URL: "%(protocol)s://%(domain)s:/grafana" + GF_SECURITY_ALLOW_EMBEDDING: "true" + depends_on: + influxdb: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "nc -z localhost 3000"] + interval: 10s + retries: 10 + volumes: + - ./grafana:/etc/grafana/provisioning + - ./grafana/dashboards:/var/lib/grafana/dashboards + deploy: + resources: + limits: + memory: 256M + + influxdb: + image: influxdb:1.8.10 + restart: unless-stopped + environment: + INFLUXDB_DB: "nussknacker_metrics" + healthcheck: + test: [ "CMD-SHELL", "influx -execute 'SHOW DATABASES'" ] + interval: 10s + retries: 10 + deploy: + resources: + limits: + memory: 128M + + ### KAFKA-related services: + + kafka: + image: bitnami/kafka:3.7.0 + restart: unless-stopped + hostname: nu-kafka + environment: + KAFKA_CFG_NODE_ID: 0 + KAFKA_CFG_PROCESS_ROLES: "controller,broker" + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "0@kafka:9093" + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" + healthcheck: + test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list" ] + interval: 10s + retries: 10 + deploy: + resources: + limits: + memory: 512M + + schema-registry: + image: bitnami/schema-registry:7.6.1 + restart: unless-stopped + environment: + SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081" + SCHEMA_REGISTRY_KAFKA_BROKERS: "PLAINTEXT://kafka:9092" + depends_on: + kafka: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", 'echo -e "GET /subjects HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" | nc localhost 8081 | grep -q "HTTP/1.1 200 OK" | exit 0'] + interval: 10s + retries: 10 + deploy: + resources: + limits: + memory: 384M + + akhq: + image: tchiotludo/akhq:0.24.0 + restart: unless-stopped + environment: + AKHQ_CONFIGURATION: | + micronaut: + server: + context-path: /akhq + akhq: + connections: + nussknacker-kafka: + properties: + bootstrap.servers: "kafka:9092" + schema-registry: + url: "http://schema-registry:8081" + depends_on: + kafka: + condition: service_healthy + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8080" ] + interval: 10s + retries: 10 + deploy: + resources: + limits: + memory: 256M + + ### FLINK-related services + + flink-jobmanager: + build: + context: ./flink/ + args: + FLINK_VERSION: "1.18.1-scala_2.12-java11" + restart: unless-stopped + command: jobmanager + environment: + JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager" + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8081/jobs/overview" ] + interval: 10s + retries: 10 + volumes: + - nussknacker_flink_data:/opt/flink/data + + flink-taskmanager: + build: + context: ./flink/ + args: + FLINK_VERSION: "1.18.1-scala_2.12-java11" + restart: unless-stopped + command: taskmanager + environment: + JOB_MANAGER_RPC_ADDRESS: "flink-jobmanager" + depends_on: + flink-jobmanager: + condition: service_healthy + telegraf: + condition: service_healthy + volumes_from: + - flink-jobmanager + deploy: + resources: + limits: + memory: 1024M + + telegraf: + image: telegraf:1.30.2 + restart: unless-stopped + healthcheck: + test: [ "CMD-SHELL", "curl -f http://localhost:8087/write" ] + interval: 10s + retries: 10 + volumes: + - ./telegraf/telegraf.conf:/etc/telegraf/telegraf.conf + deploy: + resources: + limits: + memory: 128M + +volumes: + nussknacker_designer_data: + name: nussknacker_designer_data + nussknacker_flink_data: + name: nussknacker_flink_data diff --git a/new-quickstart/download-and-start.sh b/new-quickstart/download-and-start.sh new file mode 100755 index 00000000..89d36c55 --- /dev/null +++ b/new-quickstart/download-and-start.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e + +BRANCH_NAME="main" +NU_QUICKSTART_DIR="nussknacker-quickstart-$BRANCH_NAME" + +wget -O "$NU_QUICKSTART_DIR.zip" "https://github.com/TouK/nussknacker-quickstart/archive/refs/heads/$BRANCH_NAME.zip" +rm -rf "$NU_QUICKSTART_DIR" +unzip -q "$NU_QUICKSTART_DIR.zip" +cd "$NU_QUICKSTART_DIR/new-quickstart" +./start.sh \ No newline at end of file diff --git a/new-quickstart/flink/Dockerfile b/new-quickstart/flink/Dockerfile new file mode 100644 index 00000000..e6d7cda9 --- /dev/null +++ b/new-quickstart/flink/Dockerfile @@ -0,0 +1,17 @@ +ARG FLINK_VERSION + +FROM flink:${FLINK_VERSION} + +USER root +RUN echo '#!/bin/sh' > /ex-docker-entrypoint.sh && \ + echo 'export FLINK_PROPERTIES=$(cat /opt/flink/conf/flink-properties.yml) && /docker-entrypoint.sh "$@"' >> /ex-docker-entrypoint.sh && \ + chmod +x /ex-docker-entrypoint.sh + +USER flink +COPY flink-properties.yml /opt/flink/conf/ +RUN mkdir -p /opt/flink/data && \ + chmod -R 777 /opt/flink/data + +VOLUME /opt/flink/data + +ENTRYPOINT [ "/ex-docker-entrypoint.sh" ] diff --git a/new-quickstart/flink/flink-properties.yml b/new-quickstart/flink/flink-properties.yml new file mode 100644 index 00000000..12f45da9 --- /dev/null +++ b/new-quickstart/flink/flink-properties.yml @@ -0,0 +1,28 @@ +taskmanager.numberOfTaskSlots: 8 + +state.backend: filesystem +state.checkpoints.dir: file:///opt/flink/data/checkpoints +state.savepoints.dir: file:///opt/flink/data/savepoints + +#Below are base settings for rocksdb metrics, that can be used for grafana dashboards +state.backend.rocksdb.metrics.estimate-num-keys: true +state.backend.rocksdb.metrics.estimate-live-data-size: true +state.backend.rocksdb.metrics.cur-size-all-mem-tables: true +state.backend.rocksdb.metrics.size-all-mem-tables: true +# We can have many jobs per cluster, in such setting managed memory is not easy to tune +state.backend.rocksdb.memory.managed: false +# For frequent writes increase the value as needed. Currently RocksDB settings can only be changed per Flink cluster +state.backend.rocksdb.writebuffer.size: 256m + +metrics.reporters: influxdb_reporter +metrics.reporter.influxdb_reporter.factory.class: org.apache.flink.metrics.influxdb.InfluxdbReporterFactory +metrics.reporter.influxdb_reporter.host: telegraf +metrics.reporter.influxdb_reporter.port: 8087 +metrics.reporter.influxdb_reporter.db: nussknacker_metrics +metrics.reporter.influxdb_reporter.scope.variables.excludes: tm_id;job_id;task_id;task_attempt_id;operator_id;task_attempt_num;task_name +metrics.scope.jm: local..jobmanagerGlobal +metrics.scope.jm.job: local..jobmanagerJob. +metrics.scope.tm: local..taskmanagerGlobal. +metrics.scope.tm.job: local..taskmanagerJob.. +metrics.scope.task: local..taskmanagerTask.... +metrics.scope.operator: local..taskmanagerTask.... diff --git a/new-quickstart/grafana/dashboards/dashboards.yml b/new-quickstart/grafana/dashboards/dashboards.yml new file mode 100644 index 00000000..58732b94 --- /dev/null +++ b/new-quickstart/grafana/dashboards/dashboards.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +providers: +- name: default + orgId: 1 + folder: '' + folderUid: '' + type: file + disableDeletion: true + editable: true + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/new-quickstart/grafana/dashboards/nussknacker-lite-scenario.json b/new-quickstart/grafana/dashboards/nussknacker-lite-scenario.json new file mode 100644 index 00000000..af0deb70 --- /dev/null +++ b/new-quickstart/grafana/dashboards/nussknacker-lite-scenario.json @@ -0,0 +1,2320 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": 1, + "iteration": 1638475135627, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "title": "Basics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"source.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Source throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 3, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "hide": false, + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"end.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events that passed whole scenario /s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 5, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"dead_end.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rejected events /s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 4, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"error.instantRateByNode.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "description": "IMPORTANT. \nThis metric does NOT show the overall lag on Kafka source; a largest lag among all the source partitions is shown.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 9, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_taskId]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "taskId" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + } + ], + "measurement": "records-lag-max", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max lag among Kafka source partitions (events)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 8, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay.histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "min" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - minimum", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 22 + }, + "hiddenSeries": false, + "id": 6, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay.histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - median", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 7, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay.histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - maximum", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 22 + }, + "hiddenSeries": false, + "id": 13, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay.minimalDelay", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time from latest event", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 24, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 10, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - median OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 11, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - 99th percentile OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 12, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service.OK.instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, instanceId\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - throughput OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 15, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - median FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 37 + }, + "hiddenSeries": false, + "id": 16, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - 99th percentile FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "hiddenSeries": false, + "id": 17, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_scenario - $tag_serviceName", + "dsType": "influxdb", + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service.FAIL.instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, instanceId\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - throughput FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "External services", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "regex wildcard", + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "influx", + "definition": "", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "pipe", + "name": "scenarioName", + "options": [], + "query": "SHOW TAG VALUES FROM \"source.instantRate\" WITH KEY = \"scenario\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "local", + "value": "local" + }, + "datasource": "influx", + "definition": "", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "env", + "options": [], + "query": "SHOW TAG VALUES FROM \"source.instantRate\" WITH KEY = \"env\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "description": null, + "error": null, + "hide": 0, + "label": null, + "name": "interval", + "options": [ + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": true, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s,30s,1m,3m, 10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Lite scenario", + "uid": "nussknacker-lite-scenario", + "version": 101 +} diff --git a/new-quickstart/grafana/dashboards/nussknacker-request-response-scenario.json b/new-quickstart/grafana/dashboards/nussknacker-request-response-scenario.json new file mode 100644 index 00000000..0755bc06 --- /dev/null +++ b/new-quickstart/grafana/dashboards/nussknacker-request-response-scenario.json @@ -0,0 +1,2346 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": "influx", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": "influx", + "refId": "A" + } + ], + "title": "Basics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "datasource": "influx", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"invocation.success.histogram\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 2xx - throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 3, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "datasource": "influx", + "dsType": "influxdb", + "hide": false, + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"end.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + }, + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "datasource": "influx", + "hide": false, + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"dead_end.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Outputs/s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 4, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "datasource": "influx", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"error.instantRateByNode.count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, instanceId\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 27, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"p50\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"p50\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 2xx - median", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 28, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"p99\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"p99\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 2xx - median - 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 15 + }, + "hiddenSeries": false, + "id": 31, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"max\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"max\") FROM \"invocation.success.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 2xx - max", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 29, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"p50\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"p50\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 4/5xx - median", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "hiddenSeries": false, + "id": 30, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"p99\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"p99\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 4/5xx - 99th percentile", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "hiddenSeries": false, + "id": 32, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - (all)", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"max\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\" fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + }, + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "hide": false, + "query": "SELECT max(\"max\") FROM \"invocation.failure.histogram\" WHERE (\"env\" =~ /^$env$/ AND \"scenario\" =~ /^$scenarioName$/) AND $timeFilter GROUP BY time($interval), \"scenario\", \"serviceName\", \"instanceId\" fill(null)", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Response 4/5xx - max", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": "influx", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": "influx", + "refId": "A" + } + ], + "title": "External services", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 10, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - median OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 11, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.OK.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - 99th percentile OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 12, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]]", + "datasource": "influx", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service.OK.instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, instanceId\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - throughput OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 15, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - median FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 37 + }, + "hiddenSeries": false, + "id": 16, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (instanceId [[tag_instanceId]])", + "datasource": "influx", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "instanceId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service.FAIL.histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - 99th percentile FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "hiddenSeries": false, + "id": 17, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_scenario - $tag_serviceName", + "datasource": "influx", + "dsType": "influxdb", + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service.FAIL.instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, instanceId\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Services - throughput FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "30s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "regex wildcard", + "current": { + "selected": true, + "text": [ + "test proxy process for rr" + ], + "value": [ + "test proxy process for rr" + ] + }, + "datasource": "influx", + "definition": "SHOW TAG VALUES FROM \"invocation.success.histogram\" WITH KEY = \"scenario\"", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "pipe", + "name": "scenarioName", + "options": [], + "query": "SHOW TAG VALUES FROM \"invocation.success.histogram\" WITH KEY = \"scenario\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "datasource": "influx", + "definition": "SHOW TAG VALUES FROM \"invocation.success.histogram\" WITH KEY = \"env\"", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "env", + "options": [], + "query": "SHOW TAG VALUES FROM \"invocation.success.histogram\" WITH KEY = \"env\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "hide": 0, + "name": "interval", + "options": [ + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": true, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s,30s,1m,3m, 10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "RequestResponse scenario", + "uid": "nussknacker-request-response-scenario", + "version": 1, + "weekStart": "" +} diff --git a/new-quickstart/grafana/dashboards/nussknacker-scenario.json b/new-quickstart/grafana/dashboards/nussknacker-scenario.json new file mode 100644 index 00000000..7dd01964 --- /dev/null +++ b/new-quickstart/grafana/dashboards/nussknacker-scenario.json @@ -0,0 +1,3467 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": 1, + "iteration": 1638475135627, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "title": "Basics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 2, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"source_count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, slot\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Source throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 3, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "hide": false, + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"end_count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, slot\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events that passed whole scenario /s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 5, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"dead_end_count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, slot\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rejected events /s", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 4, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(value)\nFROM (\n SELECT non_negative_derivative(last(\"count\"), 1s) AS value\n FROM \"error_instantRateByNode_count\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, nodeId, slot\n)\nGROUP BY time($__interval), scenario, nodeId\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "description": "IMPORTANT. \nThis metric does NOT show the overall lag on Kafka source; a largest lag among all the source partitions is shown.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 9, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_operator_name]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "operator_name" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + } + ], + "measurement": "records-lag-max", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max lag among Kafka source partitions (events)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 22 + }, + "hiddenSeries": false, + "id": 8, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay_histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "min" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - minimum", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 22 + }, + "hiddenSeries": false, + "id": 6, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay_histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - median", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 7, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay_histogram", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Delay since event occurrence - maximum", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 22 + }, + "hiddenSeries": false, + "id": 13, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_nodeId]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "nodeId" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "eventtimedelay_minimalDelay", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Time from latest event", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 24, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 10, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service_OK_histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - median OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 11, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service_OK_histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - 99th percentile OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 12, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]]", + "dsType": "influxdb", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service_OK_instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, slot\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - throughput OK", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 15, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service_FAIL_histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p50" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - median FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 37 + }, + "hiddenSeries": false, + "id": 16, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_serviceName]] (slot [[tag_slot]])", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "scenario" + ], + "type": "tag" + }, + { + "params": [ + "serviceName" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "service_FAIL_histogram", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "p99" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - 99th percentile FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ns", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "influx", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "hiddenSeries": false, + "id": 17, + "interval": "$interval", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_scenario - $tag_serviceName", + "dsType": "influxdb", + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(mean)\nFROM (\n SELECT mean(value)\n FROM \"service_FAIL_instantRate\"\n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), scenario, serviceName, slot\n)\nGROUP BY time($__interval), scenario, serviceName", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Services - throughput FAIL", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "External services", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 22, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Estimates of RocksDB state size. Aggregates (such as session windows) can have many internal states, which are reported separately. ", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_operator_name]] ($0)", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "operator_name" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "/^.*.cur-size-all-mem-tables/", + "orderByTime": "ASC", + "policy": "default", + "query": "\nSELECT sum(\"value\")\nFROM (\n SELECT mean(\"value\") AS \"value\"\n FROM /^.*.cur-size-all-mem-tables/ \n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter \n GROUP BY time($__interval), \"scenario\", \"operator_name\", \"slot\"\n) GROUP BY time($__interval), \"scenario\", \"operator_name\" fill(linear)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RocksDB cur-size-all-mem-tables", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Estimates of RocksDB state size. Aggregates (such as session windows) can have many internal states, which are reported separately. ", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 45 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_operator_name]] ($0)", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "operator_name" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "/^.*.cur-size-all-mem-tables/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"value\")\nFROM (\n SELECT mean(\"value\") AS \"value\"\n FROM /^.*.estimate-live-data-size/ \n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter\n GROUP BY time($__interval), \"scenario\", \"operator_name\", \"slot\"\n) GROUP BY time($__interval), \"scenario\", \"operator_name\" fill(linear)\n", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RocksDB estimate-live-data-size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Estimates on number of keys in RocksDB state. Aggregates (e.g. session windows) can have many internal states. Please remember that this is only estimate, based on internal RocksDB metrics", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 45 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_scenario]] - [[tag_operator_name]] ($0)", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "operator_name" + ], + "type": "tag" + }, + { + "params": [ + "slot" + ], + "type": "tag" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "/^.*.estimate-num-keys/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"value\")\nFROM (\n SELECT mean(\"value\") AS \"value\"\n FROM /^.*.estimate-num-keys/ \n WHERE scenario =~ /^$scenarioName$/ AND env = '$env' AND $timeFilter \n GROUP BY time($__interval), \"scenario\", \"operator_name\", \"slot\"\n) GROUP BY time($__interval), \"scenario\", \"operator_name\" fill(linear)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RocksDB estimate-num-keys", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "RocksDB state", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 28, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_uptime", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Uptime", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 32, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_fullRestarts", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Number of restarts", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 37, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_lastCheckpointDuration", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last checkpoint duration", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 38, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "linear" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_lastCheckpointSize", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Last checkpoint size", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 35, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_numberOfCompletedCheckpoints", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Completed checkpoints", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 34, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "jobmanager_job_numberOfFailedCheckpoints", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "scenario", + "operator": "=~", + "value": "/^$scenarioName$/" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed checkpoints", + "type": "stat" + } + ], + "title": "Scenario health", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "regex wildcard", + "allValue": null, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "influx", + "definition": "", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "pipe", + "name": "scenarioName", + "options": [], + "query": "SHOW TAG VALUES FROM \"source_instantRate\" WITH KEY = \"scenario\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "local", + "value": "local" + }, + "datasource": "influx", + "definition": "", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "env", + "options": [], + "query": "SHOW TAG VALUES FROM \"source_instantRate\" WITH KEY = \"env\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "description": null, + "error": null, + "hide": 0, + "label": null, + "name": "interval", + "options": [ + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": true, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s,30s,1m,3m, 10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Scenario", + "uid": "nussknacker-scenario", + "version": 101 +} \ No newline at end of file diff --git a/new-quickstart/grafana/datasources/datasources.yml b/new-quickstart/grafana/datasources/datasources.yml new file mode 100644 index 00000000..c48289b0 --- /dev/null +++ b/new-quickstart/grafana/datasources/datasources.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +datasources: +- name: influx + type: influxdb + access: proxy + orgId: 1 + url: http://influxdb:8086 + database: nussknacker_metrics + basicAuth: false + withCredentials: false + isDefault: true + version: 1 + editable: false diff --git a/new-quickstart/nginx/additional/akhq.conf b/new-quickstart/nginx/additional/akhq.conf new file mode 100644 index 00000000..548b8238 --- /dev/null +++ b/new-quickstart/nginx/additional/akhq.conf @@ -0,0 +1,3 @@ +location /akhq/ { + proxy_pass http://akhq:8080/akhq/; +} diff --git a/new-quickstart/nginx/additional/flink.conf b/new-quickstart/nginx/additional/flink.conf new file mode 100644 index 00000000..13f1fd87 --- /dev/null +++ b/new-quickstart/nginx/additional/flink.conf @@ -0,0 +1,3 @@ +location /flink/ { + proxy_pass http://flink-jobmanager:8081/; +} diff --git a/new-quickstart/nginx/nginx.conf b/new-quickstart/nginx/nginx.conf new file mode 100644 index 00000000..bf82dec8 --- /dev/null +++ b/new-quickstart/nginx/nginx.conf @@ -0,0 +1,32 @@ +http { + server { + listen 8080; + + location /grafana/ { + proxy_set_header Host $http_host; + proxy_pass http://grafana:3000/; + } + + include additional/*.conf; + + # this is the internal Docker DNS, cache only for 30s + # This setting + substitution pattern used below is because we want ot have separated compose for designer and environment. + # More details you can find here: https://sandro-keil.de/blog/let-nginx-start-if-upstream-host-is-unavailable-or-down/ + # It not works with services (e.g.) grafana (error: "Grafana has failed to load its application files") + resolver 127.0.0.11 valid=30s; + + location / { + proxy_pass http://designer:8080; + } + } + # exposing Request-Response Lite Embedded services defined in Nu + server { + listen 8181; + + location / { + proxy_pass http://designer:8181/; + } + } +} + +events {} diff --git a/new-quickstart/quickstart-setup/Dockerfile b/new-quickstart/quickstart-setup/Dockerfile new file mode 100644 index 00000000..1d630bc3 --- /dev/null +++ b/new-quickstart/quickstart-setup/Dockerfile @@ -0,0 +1,18 @@ +FROM bitnami/kafka:3.7.0 + +USER root + +RUN apt update && \ + apt install -y curl jq kafkacat less && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY entrypoint.sh / +COPY scripts/ /app/scripts/ +COPY data/ /app/data/ +COPY setup/ /app/setup/ + +HEALTHCHECK --interval=10s --timeout=1s --retries=12 CMD /bin/bash -c 'test -f "/app/healthy"' + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/new-quickstart/quickstart-setup/data/http/generate-requests/loan.sh b/new-quickstart/quickstart-setup/data/http/generate-requests/loan.sh new file mode 100755 index 00000000..768c704f --- /dev/null +++ b/new-quickstart/quickstart-setup/data/http/generate-requests/loan.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +source ../../../scripts/utils/lib.sh + +ID="$(random_4digit_number)" +AMOUNT="$(random_4digit_number)" +REQUEST_TYPE="$(pick_randomly "loan" "mortgage" "insurance")" +CITY="$(pick_randomly "Warszawa" "Berlin" "Gdańsk" "Kraków", "Poznań", "Praga")" + +echo "{\"customerId\": \"$ID\", \"requestedAmount\": $AMOUNT, \"requestType\": \"$REQUEST_TYPE\", \"location\": { \"city\": \"$CITY\", \"street\": \"\" }}" \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/data/http/slugs.txt b/new-quickstart/quickstart-setup/data/http/slugs.txt new file mode 100644 index 00000000..41d9961f --- /dev/null +++ b/new-quickstart/quickstart-setup/data/http/slugs.txt @@ -0,0 +1,2 @@ +# Request-Response OpenAPI services slugs the static and generated requests should be sent to +loan diff --git a/new-quickstart/quickstart-setup/data/http/static-requests/loan.txt b/new-quickstart/quickstart-setup/data/http/static-requests/loan.txt new file mode 100644 index 00000000..9b667512 --- /dev/null +++ b/new-quickstart/quickstart-setup/data/http/static-requests/loan.txt @@ -0,0 +1,7 @@ +# Example Request-Response OpenAPI service requests (request payload per line) +#{"customerId": "anon", "requestedAmount": 1555, "requestType": "mortgage", "location": { "city": "Warszawa", "street": "Marszałkowska" }} +#{"customerId": "anon", "requestedAmount": 86, "requestType": "loan", "location": { "city": "Lublin", "street": "Głęboka" }} +#{"customerId": "1", "requestedAmount": 1000, "requestType": "loan", "location": { "city": "Warszawa", "street": "Marszałkowska" }} +#{"customerId": "1", "requestedAmount": 500, "requestType": "savings", "location": { "city": "London", "street": "Kensington" }} +#{"customerId": "4", "requestedAmount": 2000, "requestType": "mortgage", "location": { "city": "Lublin", "street": "Lipowa" }} +#{"customerId": "3", "requestedAmount": 2000, "requestType": "loan", "location": { "city": "Lublin", "street": "Głęboka" }} diff --git a/new-quickstart/quickstart-setup/data/kafka/generate-messages/transactions.sh b/new-quickstart/quickstart-setup/data/kafka/generate-messages/transactions.sh new file mode 100755 index 00000000..d0527d84 --- /dev/null +++ b/new-quickstart/quickstart-setup/data/kafka/generate-messages/transactions.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +source ../../../scripts/utils/lib.sh + +ID=$((1 + $(random_4digit_number) % 5)) +AMOUNT=$((1 + $(random_4digit_number) % 30)) +TIME=$(($(now) - $(random_4digit_number) % 20)) + +echo "{ \"clientId\": \"Client$ID\", \"amount\": $AMOUNT, \"eventDate\": $TIME}" \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/data/kafka/static-messages/transactions.txt b/new-quickstart/quickstart-setup/data/kafka/static-messages/transactions.txt new file mode 100644 index 00000000..65a41522 --- /dev/null +++ b/new-quickstart/quickstart-setup/data/kafka/static-messages/transactions.txt @@ -0,0 +1,3 @@ +# Example messages below (message per line) +#{ "clientId": "Client1", "amount": 100, "eventDate": 1720166429}" +#{ "clientId": "Client2", "amount": 1000, "eventDate": 1720166429}" diff --git a/new-quickstart/quickstart-setup/data/kafka/topics.txt b/new-quickstart/quickstart-setup/data/kafka/topics.txt new file mode 100644 index 00000000..05a70cee --- /dev/null +++ b/new-quickstart/quickstart-setup/data/kafka/topics.txt @@ -0,0 +1,2 @@ +# Topic name the static and generated messages should be sent to +Transactions diff --git a/new-quickstart/quickstart-setup/entrypoint.sh b/new-quickstart/quickstart-setup/entrypoint.sh new file mode 100755 index 00000000..a73aedaa --- /dev/null +++ b/new-quickstart/quickstart-setup/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +rm -fr /app/healthy + +while IFS= read -r script; do + "$script" +done < <(find /app/scripts/auto-executed -type f -name '*.sh' | sort) + +echo "Setup done!" + +touch /app/healthy + +# loop forever (you can use manually called utils scripts now) +tail -f /dev/null diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/001-setup-schemas.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/001-setup-schemas.sh new file mode 100755 index 00000000..9320786c --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/001-setup-schemas.sh @@ -0,0 +1,58 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function createJsonSchema() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) schema name, 2) schema file path" + exit 11 + fi + + set -e + + local SCHEMA_NAME=$1 + local SCHEMA_FILE=$2 + + echo "Creating schema '$SCHEMA_NAME' ..." + ESCAPED_JSON_SCHEMA=$(awk 'BEGIN{ORS="\\n"} {gsub(/"/, "\\\"")} 1' < "$SCHEMA_FILE") + + local REQUEST_BODY="{ + \"schema\": \"$ESCAPED_JSON_SCHEMA\", + \"schemaType\": \"JSON\", + \"references\": [] + }" + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X POST "http://schema-registry:8081/subjects/${SCHEMA_NAME}/versions" \ + -H "Content-Type: application/vnd.schemaregistry.v1+json" -d "$REQUEST_BODY" + ) + + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + + if [[ "$HTTP_STATUS" != 200 ]] ; then + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + echo -e "Error: Cannot create schema $SCHEMA_NAME.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 12 + fi + + echo "Schema '$SCHEMA_NAME' created!" +} + +echo "Starting to add preconfigured schemas ..." + +while IFS= read -r SCHEMA_FILENAME; do + + if [[ $SCHEMA_FILENAME == "#"* ]]; then + continue + fi + + SCHEMA_NAME="$(basename "$SCHEMA_FILENAME" ".schema.json")-value" + createJsonSchema "$SCHEMA_NAME" "$(realpath ../../setup/schema-registry/schemas/"$SCHEMA_FILENAME")" + +done < "../../setup/schema-registry/active-schemas.txt" + + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/002-setup-topics.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/002-setup-topics.sh new file mode 100755 index 00000000..1e2d2953 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/002-setup-topics.sh @@ -0,0 +1,30 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function createTopic() { + if [ "$#" -ne 1 ]; then + echo "Error: One parameter required: 1) topic name" + exit 11 + fi + + set -e + local TOPIC_NAME=$1 + + echo "Creating topic '$TOPIC_NAME'" + ../utils/kafka/create-topic-idempotently.sh "$TOPIC_NAME" +} + +echo "Starting to create preconfigured topics ..." + +while IFS= read -r TOPIC_NAME; do + + if [[ $TOPIC_NAME == "#"* ]]; then + continue + fi + + createTopic "$TOPIC_NAME" + +done < "../../setup/kafka/topics.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/003-import-and-deploy-example-scenarios.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/003-import-and-deploy-example-scenarios.sh new file mode 100755 index 00000000..67b53420 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/003-import-and-deploy-example-scenarios.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function importAndDeployScenario() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) scenario name, 2) example scenario file path" + exit 11 + fi + + set -e + + local EXAMPLE_SCENARIO_NAME=$1 + local EXAMPLE_SCENARIO_FILE=$2 + + ../utils/nu/load-scenario-from-json-file.sh "$EXAMPLE_SCENARIO_NAME" "$EXAMPLE_SCENARIO_FILE" + ../utils/nu/deploy-scenario-and-wait-for-running-state.sh "$EXAMPLE_SCENARIO_NAME" +} + +echo "Starting to import and deploy example scenarios ..." + +while IFS= read -r EXAMPLE_SCENARIO_FILENAME; do + + if [[ $EXAMPLE_SCENARIO_FILENAME == "#"* ]]; then + continue + fi + + EXAMPLE_SCENARIO_NAME=$(basename "$EXAMPLE_SCENARIO_FILENAME" ".json") + + importAndDeployScenario "$EXAMPLE_SCENARIO_NAME" "$(realpath ../../setup/nu/scenarios/"$EXAMPLE_SCENARIO_FILENAME")" + +done < "../../setup/nu/examples.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/004-send-kafka-static-messages copy.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/004-send-kafka-static-messages copy.sh new file mode 100755 index 00000000..c661da35 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/004-send-kafka-static-messages copy.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function sendMessage() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) topic name, 2) message" + exit 11 + fi + + set -e + + local TOPIC_NAME=$1 + local MSG=$2 + + echo "Sending message $MSG to '$TOPIC_NAME'" + ../utils/kafka/send-to-topic.sh "$TOPIC_NAME" "$MSG" + echo "Message sent!" +} + +echo "Starting to send preconfigured messages ..." + +while IFS= read -r TOPIC_NAME; do + + if [[ $TOPIC_NAME == "#"* ]]; then + continue + fi + + MESSAGES_FILE="../../data/kafka/static-messages/$TOPIC_NAME.txt" + + if [[ -f "$MESSAGES_FILE" ]]; then + while IFS= read -r MSG; do + if [[ $MSG == "#"* ]]; then + continue + fi + + sendMessage "$TOPIC_NAME" "$MSG" + done < "$MESSAGES_FILE" + fi + +done < "../../data/kafka/topics.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/005-send-http-static-requests.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/005-send-http-static-requests.sh new file mode 100755 index 00000000..2c38214f --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/005-send-http-static-requests.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function sendRequest() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) Request-Response OpenAPI service slug, 2) request body" + exit 11 + fi + + set -e + + local OPENAPI_SERVICE_SLUG=$1 + local REQUEST_BODY=$2 + + echo "Sending request '$REQUEST_BODY' to Request-Response '$OPENAPI_SERVICE_SLUG' OpenAPI service ..." + local RESPONSE + RESPONSE=$(../utils/http/send-request-to-nu-openapi-service.sh "$OPENAPI_SERVICE_SLUG" "$REQUEST_BODY") + echo "Response: $RESPONSE" +} + +echo "Starting to send preconfigured Request-Response OpenAPI service requests ..." + +while IFS= read -r OPENAPI_SERVICE_SLUG; do + + if [[ $OPENAPI_SERVICE_SLUG == "#"* ]]; then + continue + fi + + MESSAGES_FILE="../../data/http/static-requests/$OPENAPI_SERVICE_SLUG.txt" + + if [[ -f "$MESSAGES_FILE" ]]; then + while IFS= read -r REQUEST_BODY; do + if [[ $REQUEST_BODY == "#"* ]]; then + continue + fi + + sendRequest "$OPENAPI_SERVICE_SLUG" "$REQUEST_BODY" + done < "$MESSAGES_FILE" + fi + +done < "../../data/http/slugs.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/006-continuously-send-kafka-generated-messages.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/006-continuously-send-kafka-generated-messages.sh new file mode 100755 index 00000000..d98cc65d --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/006-continuously-send-kafka-generated-messages.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function runMessageSending() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) topic name, 2) message generator script" + exit 11 + fi + + set -e + + local TOPIC_NAME=$1 + local MSG_GENERATOR_SCRIPT=$2 + + echo "Starting to send to '$TOPIC_NAME' messages generated by '$MSG_GENERATOR_SCRIPT' generator script" + + mkdir -p /var/log/continuously-send-to-topic + nohup ../utils/kafka/continuously-send-to-topic.sh "$TOPIC_NAME" "$MSG_GENERATOR_SCRIPT" > /var/log/continuously-send-to-topic/output.log 2>&1 & +} + +echo "Starting to send generated messages ..." + +while IFS= read -r TOPIC_NAME; do + + if [[ $TOPIC_NAME == "#"* ]]; then + continue + fi + + MSG_GENERATION_SCRIPT="../../data/kafka/generate-messages/$TOPIC_NAME.sh" + + if [[ -f "$MSG_GENERATION_SCRIPT" ]]; then + runMessageSending "$TOPIC_NAME" "$(realpath $MSG_GENERATION_SCRIPT)" + fi + +done < "../../data/kafka/topics.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/auto-executed/007-continuously-send-http-generated-requests.sh b/new-quickstart/quickstart-setup/scripts/auto-executed/007-continuously-send-http-generated-requests.sh new file mode 100755 index 00000000..61f0798b --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/auto-executed/007-continuously-send-http-generated-requests.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +function runRequestSending() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) OpenAPI service slug, 2) request generator script" + exit 11 + fi + + set -e + + local OPENAPI_SERVICE_SLUG=$1 + local REQUEST_GENERATOR_SCRIPT=$2 + + echo "Starting to send to '$OPENAPI_SERVICE_SLUG' OpenAPI service, requests generated by '$REQUEST_GENERATOR_SCRIPT' generator script" + + mkdir -p /var/log/continuously-send-http-requests + nohup ../utils/http/continuously-send-http-requests.sh "$OPENAPI_SERVICE_SLUG" "$REQUEST_GENERATOR_SCRIPT" > /var/log/continuously-send-http-requests/output.log 2>&1 & +} + +echo "Starting to send generated requests to Nu OpenAPI services ..." + +while IFS= read -r OPENAPI_SERVICE_SLUG; do + + if [[ $OPENAPI_SERVICE_SLUG == "#"* ]]; then + continue + fi + + REQUEST_GENERATOR_SCRIPT="../../data/http/generate-requests/$OPENAPI_SERVICE_SLUG.sh" + + if [[ -f "$REQUEST_GENERATOR_SCRIPT" ]]; then + runRequestSending "$OPENAPI_SERVICE_SLUG" "$(realpath $REQUEST_GENERATOR_SCRIPT)" + fi + +done < "../../data/http/slugs.txt" + +echo -e "DONE!\n\n" diff --git a/new-quickstart/quickstart-setup/scripts/utils/http/continuously-send-http-requests.sh b/new-quickstart/quickstart-setup/scripts/utils/http/continuously-send-http-requests.sh new file mode 100755 index 00000000..1a7f4a8a --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/http/continuously-send-http-requests.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +if [ "$#" -ne 2 ]; then + echo "Two parameters required: 1) OpenAPI service slug, 2) request generator script path" + exit 1 +fi + +cd "$(dirname "$0")" + +source ../lib.sh + +OPENAPI_SERVICE_SLUG=$1 +REQUEST_GENERATOR_SCRIPT=$2 + +verifyBashScript "$REQUEST_GENERATOR_SCRIPT" + +while true; do + sleep 0.1 + ./send-request-to-nu-openapi-service.sh "$OPENAPI_SERVICE_SLUG" "$($REQUEST_GENERATOR_SCRIPT)" > /dev/null || true +done diff --git a/new-quickstart/quickstart-setup/scripts/utils/http/send-request-to-nu-openapi-service.sh b/new-quickstart/quickstart-setup/scripts/utils/http/send-request-to-nu-openapi-service.sh new file mode 100755 index 00000000..debbd89c --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/http/send-request-to-nu-openapi-service.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e + +if [ "$#" -ne 2 ]; then + echo "Two parameters required: 1) Nu OpenAPI service slug, 2) request payload" + exit 1 +fi + +cd "$(dirname "$0")" + +OPENAPI_SERVICE_SLUG=$1 +REQUEST_BODY=$2 + +RESPONSE=$(curl -s -L -w "\n%{http_code}" \ + -X POST "http://nginx:8181/scenario/$OPENAPI_SERVICE_SLUG" \ + -H "Content-Type: application/json" -d "$REQUEST_BODY" +) + +HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) +RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + +if [[ "$HTTP_STATUS" != 200 ]] ; then + echo -e "'$OPENAPI_SERVICE_SLUG' OpenAPI service unexpected response.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 12 +fi + +echo "$RESPONSE_BODY" \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/scripts/utils/kafka/continuously-send-to-topic.sh b/new-quickstart/quickstart-setup/scripts/utils/kafka/continuously-send-to-topic.sh new file mode 100755 index 00000000..c96959ba --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/kafka/continuously-send-to-topic.sh @@ -0,0 +1,20 @@ +#!/bin/bash -e + +if [ "$#" -ne 2 ]; then + echo "Two parameters required: 1) topic name, 2) generator script path" + exit 1 +fi + +cd "$(dirname "$0")" + +source ../lib.sh + +TOPIC=$1 +GENERATOR_SCRIPT=$2 + +verifyBashScript "$GENERATOR_SCRIPT" + +while true; do + sleep 0.1 + ./send-to-topic.sh "$TOPIC" "$($GENERATOR_SCRIPT)" || true +done diff --git a/new-quickstart/quickstart-setup/scripts/utils/kafka/create-topic-idempotently.sh b/new-quickstart/quickstart-setup/scripts/utils/kafka/create-topic-idempotently.sh new file mode 100755 index 00000000..198d4147 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/kafka/create-topic-idempotently.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +if [ "$#" -ne 1 ]; then + echo "One parameter required: 1) topic name" + exit 1 +fi + +cd "$(dirname "$0")" + +TOPIC_NAME=$1 + +/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server kafka:9092 --if-not-exists --topic "$TOPIC_NAME" \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/scripts/utils/kafka/purge-topic.sh b/new-quickstart/quickstart-setup/scripts/utils/kafka/purge-topic.sh new file mode 100755 index 00000000..c925330e --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/kafka/purge-topic.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +if [ "$#" -ne 1 ]; then + echo "One parameter required: 1) topic name" + exit 1 +fi + +cd "$(dirname "$0")" + +TOPIC_NAME=$1 +DELETE_TOPIC_ORDER_FILE="/tmp/delete-$TOPIC_NAME.json" + +trap 'rm "$DELETE_TOPIC_ORDER_FILE"' EXIT + +echo "{ \"partitions\": [{ \"topic\": \"$TOPIC_NAME\", \"partition\": 0, \"offset\": -1 }], \"version\": 1 }" > "$DELETE_TOPIC_ORDER_FILE" + +/opt/bitnami/kafka/bin/kafka-delete-records.sh --bootstrap-server kafka:9092 -offset-json-file "$DELETE_TOPIC_ORDER_FILE" diff --git a/new-quickstart/quickstart-setup/scripts/utils/kafka/read-from-topic.sh b/new-quickstart/quickstart-setup/scripts/utils/kafka/read-from-topic.sh new file mode 100755 index 00000000..7501eb07 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/kafka/read-from-topic.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +if [ "$#" -ne 1 ]; then + echo "One parameter required: 1) topic name" + exit 1 +fi + +cd "$(dirname "$0")" + +TOPIC_NAME=$1 + +kcat -C -b kafka:9092 -t "$TOPIC_NAME" -o beginning -e -q diff --git a/new-quickstart/quickstart-setup/scripts/utils/kafka/send-to-topic.sh b/new-quickstart/quickstart-setup/scripts/utils/kafka/send-to-topic.sh new file mode 100755 index 00000000..0d387168 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/kafka/send-to-topic.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +if [ "$#" -ne 2 ]; then + echo "Two parameters required: 1) topic name, 2) message" + exit 1 +fi + +cd "$(dirname "$0")" + +TOPIC=$1 +MESSAGE=$2 + +echo "$MESSAGE" | /opt/bitnami/kafka/bin/kafka-console-producer.sh --topic "$TOPIC" --bootstrap-server kafka:9092 diff --git a/new-quickstart/quickstart-setup/scripts/utils/lib.sh b/new-quickstart/quickstart-setup/scripts/utils/lib.sh new file mode 100755 index 00000000..00d7e439 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/lib.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +function verifyBashScript() { + local FILE=$1 + + if [[ -f "$FILE" ]]; then + if [[ $(head -n 1 "$FILE") =~ ^#!/bin/bash ]]; then + return 0 + else + echo "$FILE exists but is not a Bash script." + return 1 + fi + else + echo "$FILE does not exist." + return 2 + fi +} + +function random_4digit_number() { + od -An -t d -N 2 /dev/urandom | head -n 1 | tr -d ' ' | head -c 4 +} + +function random_3digit_number() { + random_4digit_number | head -c 3 +} + +function now() { + echo "$(date +%s)$(random_3digit_number)" +} + +function pick_randomly() { + local options=("$@") + local count=${#options[@]} + local random_index=$((RANDOM % count)) + echo "${options[$random_index]}" +} \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/scripts/utils/nu/deploy-scenario-and-wait-for-running-state.sh b/new-quickstart/quickstart-setup/scripts/utils/nu/deploy-scenario-and-wait-for-running-state.sh new file mode 100755 index 00000000..17f3a67c --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/nu/deploy-scenario-and-wait-for-running-state.sh @@ -0,0 +1,96 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +if [ "$#" -lt 1 ]; then + echo "Error: One parameter required: 1) scenario name" + exit 1 +fi + +SCENARIO_NAME=$1 +TIMEOUT_SECONDS=${2:-60} +WAIT_INTERVAL=5 + +function deployScenario() { + if [ "$#" -ne 1 ]; then + echo "Error: One parameter required: 1) scenario name" + exit 11 + fi + + set -e + + local SCENARIO_NAME=$1 + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X POST "http://nginx:8080/api/processManagement/deploy/$SCENARIO_NAME" + ) + + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + + if [ "$HTTP_STATUS" != "200" ]; then + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + echo -e "Error: Cannot run scenario $SCENARIO_NAME deployment.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 12 + fi + + echo "Scenario $SCENARIO_NAME deployment started ..." +} + +function checkDeploymentStatus() { + if [ "$#" -ne 1 ]; then + echo "Error: One parameter required: 1) scenario name" + exit 21 + fi + + set -e + + local SCENARIO_NAME=$1 + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X GET "http://nginx:8080/api/processes/$SCENARIO_NAME/status" + ) + + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + + if [ "$HTTP_STATUS" != "200" ]; then + echo -e "Error: Cannot check scenario $SCENARIO_NAME deployment status.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 22 + fi + + local SCENARIO_STATUS + SCENARIO_STATUS=$(echo "$RESPONSE_BODY" | jq -r '.status.name') + echo "$SCENARIO_STATUS" +} + +echo "Deploying scenario $SCENARIO_NAME ..." + +START_TIME=$(date +%s) +END_TIME=$((START_TIME + TIMEOUT_SECONDS)) + +deployScenario "$SCENARIO_NAME" + +while true; do + DEPLOYMENT_STATUS=$(checkDeploymentStatus "$SCENARIO_NAME") + + if [ "$DEPLOYMENT_STATUS" == "RUNNING" ]; then + break + fi + + CURRENT_TIME=$(date +%s) + if [ $CURRENT_TIME -gt $END_TIME ]; then + echo "Error: Timeout for waiting for the RUNNING state of $SCENARIO_NAME deployment reached!" + exit 2 + fi + + echo "$SCENARIO_NAME deployment state is $DEPLOYMENT_STATUS. Checking again in $WAIT_INTERVAL seconds..." + sleep $WAIT_INTERVAL +done + +echo "Scenario $SCENARIO_NAME is RUNNING!" diff --git a/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json-file.sh b/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json-file.sh new file mode 100755 index 00000000..e5e0378a --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json-file.sh @@ -0,0 +1,161 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +if [ "$#" -lt 2 ]; then + echo "Error: Two parameters required: 1) scenario name, 2) scenario file path" + exit 1 +fi + +SCENARIO_NAME=$1 +SCENARIO_FILE_PATH=$2 +CATEGORY=${3:-"Default"} + +if [ ! -f "$SCENARIO_FILE_PATH" ]; then + echo "Error: Cannot find file $SCENARIO_FILE_PATH with scenario" + exit 2 +fi + +function createEmptyScenario() { + if [ "$#" -ne 4 ]; then + echo "Error: Four parameters required: 1) scenario name, 2) processing mode, 3) category, 4) engine" + exit 11 + fi + + set -e + + local SCENARIO_NAME=$1 + local PROCESSING_MODE=$2 + local CATEGORY=$3 + local ENGINE=$4 + + local REQUEST_BODY="{ + \"name\": \"$SCENARIO_NAME\", + \"processingMode\": \"$PROCESSING_MODE\", + \"category\": \"$CATEGORY\", + \"engineSetupName\": \"$ENGINE\", + \"isFragment\": false + }" + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X POST "http://nginx:8080/api/processes" \ + -H "Content-Type: application/json" -d "$REQUEST_BODY" + ) + + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + + if [ "$HTTP_STATUS" == "400" ]; then + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + + if [[ "$RESPONSE_BODY" == *"already exists"* ]]; then + echo "Scenario already exists." + exit 0 + else + echo -e "Error: Cannot create empty scenario $SCENARIO_NAME.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 12 + fi + elif [ "$HTTP_STATUS" != "201" ]; then + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + echo -e "Error: Cannot create empty scenario $SCENARIO_NAME.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 12 + fi + + echo "Empty scenario $SCENARIO_NAME created successfully." +} + +function importScenarioFromFile() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) scenario name, 2) scenario file path" + exit 21 + fi + + set -e + + local SCENARIO_NAME=$1 + local SCENARIO_FILE=$2 + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X POST "http://nginx:8080/api/processes/import/$SCENARIO_NAME" \ + -F "process=@$SCENARIO_FILE" + ) + + # Check response body and status code + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + + if [ "$HTTP_STATUS" == "200" ]; then + local SCENARIO_GRAPH + SCENARIO_GRAPH=$(echo "$RESPONSE_BODY" | jq '.scenarioGraph') + echo "$SCENARIO_GRAPH" + else + echo -e "Error: Cannot import scenario $SCENARIO_NAME.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 22 + fi +} + +function saveScenario() { + if [ "$#" -ne 2 ]; then + echo "Error: Two parameters required: 1) scenario name, 2) scenario graph JSON representation" + exit 31 + fi + + set -e + + local SCENARIO_NAME=$1 + local SCENARIO_GRAPH_JSON=$2 + + local REQUEST_BODY="{ + \"scenarioGraph\": $SCENARIO_GRAPH_JSON, + \"comment\": \"\" + }" + + local RESPONSE + RESPONSE=$(curl -s -L -w "\n%{http_code}" -u admin:admin \ + -X PUT "http://nginx:8080/api/processes/$SCENARIO_NAME" \ + -H "Content-Type: application/json" -d "$REQUEST_BODY" + ) + + local HTTP_STATUS + HTTP_STATUS=$(echo "$RESPONSE" | tail -n 1) + + if [ "$HTTP_STATUS" != "200" ]; then + local RESPONSE_BODY + RESPONSE_BODY=$(echo "$RESPONSE" | sed \$d) + echo -e "Error: Cannot save scenario $SCENARIO_NAME.\nHTTP status: $HTTP_STATUS, response body: $RESPONSE_BODY" + exit 32 + fi + + echo "Scenario $SCENARIO_NAME saved successfully." +} + +META_DATA_TYPE=$(jq -r .metaData.additionalFields.metaDataType < "$SCENARIO_FILE_PATH") +case "$META_DATA_TYPE" in + "StreamMetaData") + ENGINE="Flink" + PROCESSING_MODE="Unbounded-Stream" + ;; + "LiteStreamMetaData") + ENGINE="Lite Embedded" + PROCESSING_MODE="Unbounded-Stream" + ;; + "RequestResponseMetaData") + ENGINE="Lite Embedded" + PROCESSING_MODE="Request-Response" + ;; + *) + echo "Error: Cannot import scenario with metadata type: $META_DATA_TYPE" + exit 2 + ;; +esac + +createEmptyScenario "$SCENARIO_NAME" "$PROCESSING_MODE" "$CATEGORY" "$ENGINE" +SCENARIO_GRAPH=$(importScenarioFromFile "$SCENARIO_NAME" "$SCENARIO_FILE_PATH") +saveScenario "$SCENARIO_NAME" "$SCENARIO_GRAPH" diff --git a/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json.sh b/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json.sh new file mode 100755 index 00000000..8e80c5e7 --- /dev/null +++ b/new-quickstart/quickstart-setup/scripts/utils/nu/load-scenario-from-json.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +cd "$(dirname "$0")" + +if [ "$#" -lt 2 ]; then + echo "Error: Two parameters required: 1) scenario name, 2) scenario JSON" + exit 1 +fi + +SCENARIO_NAME=$1 +SCENARIO_JSON=$2 +SCENARIO_JSON_FILE="/tmp/scenario-$SCENARIO_NAME.json" + +echo "$SCENARIO_JSON" > "$SCENARIO_JSON_FILE" +trap 'rm "$SCENARIO_JSON_FILE"' EXIT + +./load-scenario-from-json-file.sh "$SCENARIO_NAME" "$SCENARIO_JSON_FILE" diff --git a/new-quickstart/quickstart-setup/setup/kafka/topics.txt b/new-quickstart/quickstart-setup/setup/kafka/topics.txt new file mode 100644 index 00000000..adc69ddf --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/kafka/topics.txt @@ -0,0 +1,5 @@ +# List of topics that should be created +Customers +ProcessedTransactions +SmsesWithOffer +Transactions diff --git a/new-quickstart/quickstart-setup/setup/nu/examples.txt b/new-quickstart/quickstart-setup/setup/nu/examples.txt new file mode 100644 index 00000000..3cb0625d --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/nu/examples.txt @@ -0,0 +1,4 @@ +# Scenarios to put as examples (file with scenario from the scenarios folder per line) +DetectLargeTransactions.json +#DetermineOfferedPlan.json +LoanRequest.json diff --git a/new-quickstart/quickstart-setup/setup/nu/scenarios/DetectLargeTransactions.json b/new-quickstart/quickstart-setup/setup/nu/scenarios/DetectLargeTransactions.json new file mode 100644 index 00000000..5523b1af --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/nu/scenarios/DetectLargeTransactions.json @@ -0,0 +1,125 @@ +{ + "metaData": { + "id": "DetectLargeTransactions", + "additionalFields": { + "description": null, + "properties": { + "parallelism": "1", + "spillStateToDisk": "true" + }, + "metaDataType": "StreamMetaData" + } + }, + "nodes": [ + { + "id": "transactions", + "ref": { + "typ": "kafka", + "parameters": [ + { + "name": "Topic", + "expression": { + "language": "spel", + "expression": "'Transactions'" + } + }, + { + "name": "Schema version", + "expression": { + "language": "spel", + "expression": "'latest'" + } + } + ] + }, + "additionalFields": { + "description": null, + "layoutData": { + "x": 0, + "y": 0 + } + }, + "type": "Source" + }, + { + "nextFalse": [ + ], + "id": "only large ones", + "expression": { + "language": "spel", + "expression": "#input.amount > 20" + }, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 0, + "y": 180 + } + }, + "type": "Filter" + }, + { + "id": "send for audit", + "ref": { + "typ": "kafka", + "parameters": [ + { + "name": "Topic", + "expression": { + "language": "spel", + "expression": "'ProcessedTransactions'" + } + }, + { + "name": "Schema version", + "expression": { + "language": "spel", + "expression": "'latest'" + } + }, + { + "name": "Key", + "expression": { + "language": "spel", + "expression": "" + } + }, + { + "name": "Raw editor", + "expression": { + "language": "spel", + "expression": "true" + } + }, + { + "name": "Value validation mode", + "expression": { + "language": "spel", + "expression": "'strict'" + } + }, + { + "name": "Value", + "expression": { + "language": "spel", + "expression": "#input" + } + } + ] + }, + "endResult": null, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 0, + "y": 360 + } + }, + "type": "Sink" + } + ], + "additionalBranches": [ + ] +} diff --git a/new-quickstart/quickstart-setup/setup/nu/scenarios/DetermineOfferedPlan.json b/new-quickstart/quickstart-setup/setup/nu/scenarios/DetermineOfferedPlan.json new file mode 100644 index 00000000..c73aad21 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/nu/scenarios/DetermineOfferedPlan.json @@ -0,0 +1,190 @@ +{ + "metaData" : { + "id" : "DetermineOfferedPlan", + "additionalFields": { + "description": null, + "properties": { + "parallelism": "1", + "spillStateToDisk": "true" + }, + "metaDataType": "StreamMetaData" + } + }, + "nodes" : [ + { + "id" : "Customers", + "ref" : { + "typ" : "kafka", + "parameters" : [ + { + "name" : "Topic", + "expression" : { + "language" : "spel", + "expression" : "'Customers'" + } + }, + { + "name" : "Schema version", + "expression" : { + "language" : "spel", + "expression" : "'latest'" + } + } + ] + }, + "additionalFields" : { + "description" : null, + "layoutData" : { + "x" : 181, + "y" : 10 + } + }, + "type" : "Source" + }, + { + "id" : "DecisionTable", + "service" : { + "id" : "decision-table", + "parameters" : [ + { + "name" : "Decision Table", + "expression" : { + "language" : "tabularDataDefinition", + "expression" : "{\n \"rows\": [\n [\n null,\n \"17\",\n null,\n null,\n \"Junior Package\"\n ],\n [\n \"18\",\n \"30\",\n \"Male\",\n \"true\",\n \"Premium Data Plan\"\n ],\n [\n \"18\",\n \"30\",\n \"Male\",\n \"false\",\n \"Standard Data Plan\"\n ],\n [\n \"18\",\n \"30\",\n \"Female\",\n \"true\",\n \"Premium Data Plan\"\n ],\n [\n \"18\",\n \"30\",\n \"Female\",\n \"false\",\n \"Standard Data Plan\"\n ],\n [\n \"31\",\n \"50\",\n \"Male\",\n \"true\",\n \"Family Package\"\n ],\n [\n \"31\",\n \"50\",\n null,\n \"false\",\n \"Standard Data Plan\"\n ],\n [\n \"51\",\n null,\n null,\n null,\n \"Senior Citizen Plan\"\n ]\n ],\n \"columns\": [\n {\n \"name\": \"MinAge\",\n \"type\": \"java.lang.Integer\"\n },\n {\n \"name\": \"MaxAge\",\n \"type\": \"java.lang.Integer\"\n },\n {\n \"name\": \"Gender\",\n \"type\": \"java.lang.String\"\n },\n {\n \"name\": \"IsBigSpender\",\n \"type\": \"java.lang.Boolean\"\n },\n {\n \"name\": \"OfferedPlan\",\n \"type\": \"java.lang.String\"\n }\n ]\n}" + } + }, + { + "name" : "Match condition", + "expression" : { + "language" : "spel", + "expression" : "(#ROW.MinAge != null ? #input.age > #ROW.MinAge : true) && \n(#ROW.MaxAge != null ? #input.age <= #ROW.MaxAge : true) && \n(#ROW.Gender != null ? #input.gender == #ROW.Gender : true) && \n(#ROW.IsBigSpender != null ? #input.isBigSpender == #ROW.IsBigSpender : true)" + } + } + ] + }, + "output" : "offeredPlans", + "additionalFields" : { + "description" : "Plans assigned to customers", + "layoutData" : { + "x" : 180, + "y" : 180 + } + }, + "type" : "Enricher" + }, + { + "id" : "OfferedPlansOutput", + "varName" : "outputVar", + "value" : { + "language" : "spel", + "expression" : "#offeredPlans" + }, + "additionalFields" : { + "description" : "list of cars that need service", + "layoutData" : { + "x" : 180, + "y" : 360 + } + }, + "type" : "Variable" + }, + { + "nextFalse" : [ + { + "id" : "SendPremiumPlanSMS", + "ref" : { + "typ" : "kafka", + "parameters" : [ + { + "name" : "Topic", + "expression" : { + "language" : "spel", + "expression" : "'SmsesWithOffer'" + } + }, + { + "name" : "Schema version", + "expression" : { + "language" : "spel", + "expression" : "'latest'" + } + }, + { + "name" : "Key", + "expression" : { + "language" : "spel", + "expression" : "" + } + }, + { + "name" : "Raw editor", + "expression" : { + "language" : "spel", + "expression" : "false" + } + }, + { + "name" : "assignedOffer", + "expression" : { + "language" : "spel", + "expression" : "#offeredPlans[0].OfferedPlan" + } + }, + { + "name" : "name", + "expression" : { + "language" : "spel", + "expression" : "#input.name" + } + } + ] + }, + "endResult" : null, + "isDisabled" : null, + "additionalFields" : { + "description" : null, + "layoutData" : { + "x" : 446, + "y" : 708 + } + }, + "type" : "Sink" + } + ], + "id" : "IsStandardPlan", + "expression" : { + "language" : "spel", + "expression" : "#offeredPlans[0].OfferedPlan == \"Standard Data Plan\"" + }, + "isDisabled" : null, + "additionalFields" : { + "description" : null, + "layoutData" : { + "x" : 180, + "y" : 540 + } + }, + "type" : "Filter" + }, + { + "id" : "dead-end", + "ref" : { + "typ" : "dead-end", + "parameters" : [ + ] + }, + "endResult" : null, + "isDisabled" : null, + "additionalFields" : { + "description" : null, + "layoutData" : { + "x" : -73, + "y" : 714 + } + }, + "type" : "Sink" + } + ], + "additionalBranches" : [ + ] +} diff --git a/new-quickstart/quickstart-setup/setup/nu/scenarios/LoanRequest.json b/new-quickstart/quickstart-setup/setup/nu/scenarios/LoanRequest.json new file mode 100644 index 00000000..a18e4079 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/nu/scenarios/LoanRequest.json @@ -0,0 +1,239 @@ +{ + "metaData": { + "id": "LoanRequest", + "additionalFields": { + "description": null, + "properties": { + "inputSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"customerId\": {\n \"type\": \"string\"\n },\n \"location\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n },\n \"street\": {\n \"type\": \"string\"\n }\n }\n },\n \"requestType\": {\n \"type\": \"string\"\n },\n \"requestedAmount\": {\n \"type\": \"number\"\n }\n },\n \"required\": [\"customerId\", \"location\", \"requestType\", \"requestedAmount\"],\n \"additionalProperties\": false\n}", + "outputSchema": "{\n \"type\": \"object\",\n \"properties\": {\n \"acceptedAmount\": {\n \"type\": \"number\",\n \"description\": \"Accepted amount\"\n },\n \"message\": {\n \"type\": \"string\",\n \"description\": \"Additional message\"\n }\n },\n \"required\": [\"acceptedAmount\", \"message\"],\n \"additionalProperties\": false\n}", + "slug": "loan" + }, + "metaDataType": "RequestResponseMetaData" + } + }, + "nodes": [ + { + "id": "request", + "ref": { + "typ": "request", + "parameters": [ + ] + }, + "additionalFields": { + "description": null, + "layoutData": { + "x": 360, + "y": 0 + } + }, + "type": "Source" + }, + { + "defaultNext": [ + ], + "nexts": [ + { + "expression": { + "language": "spel", + "expression": "#input.requestType == 'loan'" + }, + "nodes": [ + { + "id": "loan response", + "ref": { + "typ": "response", + "parameters": [ + { + "name": "acceptedAmount", + "expression": { + "language": "spel", + "expression": "50" + } + }, + { + "name": "message", + "expression": { + "language": "spel", + "expression": "'only small amount available'" + } + } + ] + }, + "endResult": null, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 0, + "y": 360 + } + }, + "type": "Sink" + } + ] + }, + { + "expression": { + "language": "spel", + "expression": "#input.requestType == 'mortgage'" + }, + "nodes": [ + { + "defaultNext": [ + ], + "nexts": [ + { + "expression": { + "language": "spel", + "expression": "#input.location.city == 'Warszawa'" + }, + "nodes": [ + { + "id": "Warsaw mortgage", + "ref": { + "typ": "response", + "parameters": [ + { + "name": "acceptedAmount", + "expression": { + "language": "spel", + "expression": "1000" + } + }, + { + "name": "message", + "expression": { + "language": "spel", + "expression": "'Large sum for Warszawa'" + } + } + ] + }, + "endResult": null, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 180, + "y": 540 + } + }, + "type": "Sink" + } + ] + }, + { + "expression": { + "language": "spel", + "expression": "true" + }, + "nodes": [ + { + "id": "Other city mortgage", + "ref": { + "typ": "response", + "parameters": [ + { + "name": "Raw editor", + "expression": { + "language": "spel", + "expression": "false" + } + }, + { + "name": "acceptedAmount", + "expression": { + "language": "spel", + "expression": "100" + } + }, + { + "name": "message", + "expression": { + "language": "spel", + "expression": "'Large sum for other city'" + } + } + ] + }, + "endResult": null, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 540, + "y": 540 + } + }, + "type": "Sink" + } + ] + } + ], + "id": "switch", + "additionalFields": { + "description": null, + "layoutData": { + "x": 360, + "y": 360 + } + }, + "type": "Switch" + } + ] + }, + { + "expression": { + "language": "spel", + "expression": "true" + }, + "nodes": [ + { + "id": "unknown", + "ref": { + "typ": "response", + "parameters": [ + { + "name": "acceptedAmount", + "expression": { + "language": "spel", + "expression": "0" + } + }, + { + "name": "message", + "expression": { + "language": "spel", + "expression": "'Unknown loan type'" + } + } + ] + }, + "endResult": null, + "isDisabled": null, + "additionalFields": { + "description": null, + "layoutData": { + "x": 720, + "y": 360 + } + }, + "type": "Sink" + } + ] + } + ], + "id": "loan type", + "additionalFields": { + "description": null, + "layoutData": { + "x": 360, + "y": 180 + } + }, + "type": "Switch" + } + ], + "additionalBranches": [ + ] + } \ No newline at end of file diff --git a/new-quickstart/quickstart-setup/setup/schema-registry/active-schemas.txt b/new-quickstart/quickstart-setup/setup/schema-registry/active-schemas.txt new file mode 100644 index 00000000..ac31c52c --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/schema-registry/active-schemas.txt @@ -0,0 +1,5 @@ +# JSON schemas to load (JSON schema file name from the schemas folder per line) +Customers.schema.json +ProcessedTransactions.schema.json +SmsesWithOffer.schema.json +Transactions.schema.json diff --git a/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Customers.schema.json b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Customers.schema.json new file mode 100644 index 00000000..109af2e1 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Customers.schema.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "title": "Person", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": [ + "name" + ], + "properties": { + "gender": { + "description": "The person's gender", + "type": "string", + "enum": [ + "Male", + "Female" + ] + }, + "name": { + "type": "string", + "description": "The person's full name" + }, + "isBigSpender": { + "description": "Indicates if the person is a big spender", + "type": "boolean" + }, + "age": { + "type": "integer", + "description": "The person's age in years" + } + } +} diff --git a/new-quickstart/quickstart-setup/setup/schema-registry/schemas/ProcessedTransactions.schema.json b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/ProcessedTransactions.schema.json new file mode 100644 index 00000000..ca3c0d02 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/ProcessedTransactions.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "clientId": { "type": "string" }, + "amount": { "type": "integer" }, + "isLast": { "type": "boolean", "default": false }, + "eventDate": { "type": "integer" } + }, + "required": ["clientId", "amount"], + "additionalProperties": false +} diff --git a/new-quickstart/quickstart-setup/setup/schema-registry/schemas/SmsesWithOffer.schema.json b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/SmsesWithOffer.schema.json new file mode 100644 index 00000000..7a928e75 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/SmsesWithOffer.schema.json @@ -0,0 +1,19 @@ +{ + "type": "object", + "title": "OfferAssignment", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": [ + "assignedOffer", + "name" + ], + "properties": { + "assignedOffer": { + "type": "string", + "description": "The assigned offer for the person" + }, + "name": { + "type": "string", + "description": "The person's full name" + } + } +} diff --git a/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Transactions.schema.json b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Transactions.schema.json new file mode 100644 index 00000000..ca3c0d02 --- /dev/null +++ b/new-quickstart/quickstart-setup/setup/schema-registry/schemas/Transactions.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "clientId": { "type": "string" }, + "amount": { "type": "integer" }, + "isLast": { "type": "boolean", "default": false }, + "eventDate": { "type": "integer" } + }, + "required": ["clientId", "amount"], + "additionalProperties": false +} diff --git a/new-quickstart/start.sh b/new-quickstart/start.sh new file mode 100755 index 00000000..d8924e9f --- /dev/null +++ b/new-quickstart/start.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")" + +echo -e " + +███ ██ ██ ██ ███████ ███████ ██ ██ ███ ██ █████ ██████ ██ ██ ███████ ██████ +████ ██ ██ ██ ██ ██ ██ ██ ████ ██ ██ ██ ██ ██ ██ ██ ██ ██ +██ ██ ██ ██ ██ ███████ ███████ █████ ██ ██ ██ ███████ ██ █████ █████ ██████ +██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ +██ ████ ██████ ███████ ███████ ██ ██ ██ ████ ██ ██ ██████ ██ ██ ███████ ██ ██ + + QUICKSTART + +" + +echo "Running Nussknacker Quickstart ..." +echo "" + +if ! docker version &>/dev/null; then + echo "No Docker found. Docker is required to run this Quickstart. See https://docs.docker.com/engine/install/" + exit 1 +fi + +if ! docker compose version &>/dev/null; then + echo "No docker compose found. It seems you have to upgrade your Docker installation. See https://docs.docker.com/engine/install/" + exit 2 +fi + +if ! docker compose config > /dev/null; then + echo "Cannot validate docker compose configuration. It seems you have to upgrade your Docker installation. See https://docs.docker.com/engine/install/" + exit 3 +fi + +docker compose up -d --build --wait + +echo "" +echo "Nussknacker and its dependencies are up and running." +echo "Open http://localhost:8080 and log in as admin:admin ..." diff --git a/new-quickstart/stop-and-clean.sh b/new-quickstart/stop-and-clean.sh new file mode 100755 index 00000000..4d65bb9e --- /dev/null +++ b/new-quickstart/stop-and-clean.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -e + +cd "$(dirname "$0")" + +echo "Running Nussknacker Quickstart clean up ..." +echo "" + +if ! docker version &>/dev/null; then + echo "No Docker found. Docker is required to run this Quickstart. See https://docs.docker.com/engine/install/" + exit 1 +fi + +if ! docker compose version &>/dev/null; then + echo "No docker compose found. It seems you have to upgrade your Docker installation. See https://docs.docker.com/engine/install/" + exit 2 +fi + +if ! docker compose config > /dev/null; then + echo "Cannot validate docker compose configuration. It seems you have to upgrade your Docker installation. See https://docs.docker.com/engine/install/" + exit 3 +fi + +docker compose down -v + +echo "All is cleaned. Goodbye" diff --git a/new-quickstart/telegraf/telegraf.conf b/new-quickstart/telegraf/telegraf.conf new file mode 100644 index 00000000..8a440d10 --- /dev/null +++ b/new-quickstart/telegraf/telegraf.conf @@ -0,0 +1,52 @@ +# Flink reporter for InfluxDB is somewhat limited: +# - no possibility of adding own tags +# - metric name has all tags encoded inside + +[global_tags] + env = "local" + +[[inputs.influxdb_listener]] + service_address = "0.0.0.0:8087" + read_timeout = "7s" + write_timeout = "7s" + +[[processors.rename]] + + [[processors.rename.replace]] + tag = "job_name" + dest = "scenario" + + [[processors.rename.replace]] + tag = "subtask_index" + dest = "slot" + +[[processors.strings]] + [[processors.strings.replace]] + measurement = "*" + old = "taskmanager_job_task_operator_" + new = "" + + [[processors.strings.replace]] + measurement = "*" + old = "nodeId_" + new = "" + + [[processors.strings.replace]] + measurement = "*" + old = "serviceName_" + new = "" + +[[outputs.influxdb]] + urls = ["http://influxdb:8086"] + skip_database_creation = true + database = "nussknacker_metrics" + +# Use for debugging +##[[outputs.file]] +# files = ["stdout"] + +[agent] + metric_batch_size = 10000 + metric_buffer_limit = 100000 + interval = "10s" + flush_interval = "10s"