diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7ff53d2e..07dc3ab017 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ the release. ([#2070](https://github.com/open-telemetry/opentelemetry-demo/pull/2070)) * [frontend] Cypress tests base image updated to 14.0.3 ([#2072](https://github.com/open-telemetry/opentelemetry-demo/pull/2072)) +* [grafana] Update dashboards with service map + ([#2085](https://github.com/open-telemetry/opentelemetry-demo/pull/2085)) ## 2.0.0 diff --git a/docker-compose.minimal.yml b/docker-compose.minimal.yml index 0b50e6a490..a52755b0cc 100644 --- a/docker-compose.minimal.yml +++ b/docker-compose.minimal.yml @@ -617,6 +617,7 @@ services: - OTEL_COLLECTOR_HOST - OTEL_COLLECTOR_PORT_GRPC - OTEL_COLLECTOR_PORT_HTTP + - GOMEMLIMIT=160MiB # Prometheus prometheus: diff --git a/docker-compose.yml b/docker-compose.yml index 5bb59ac14f..7013b1028f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -763,6 +763,7 @@ services: - OTEL_COLLECTOR_HOST - OTEL_COLLECTOR_PORT_GRPC - OTEL_COLLECTOR_PORT_HTTP + - GOMEMLIMIT=160MiB # Prometheus prometheus: diff --git a/src/grafana/grafana.ini b/src/grafana/grafana.ini index 2443041290..0a653bde68 100644 --- a/src/grafana/grafana.ini +++ b/src/grafana/grafana.ini @@ -373,7 +373,7 @@ serve_from_sub_path = true ;default_theme = dark # Path to a custom home page. Users are only redirected to this if the default home dashboard is used. It should match a frontend route and contain a leading slash. -; home_page = +home_page = /grafana/dashboards/f/demo/?orgId=1 # External user management, these options affect the organization users view ;external_manage_link_url = @@ -436,7 +436,7 @@ disable_login_form = true #################################### Anonymous Auth ###################### [auth.anonymous] # enable anonymous access -enabled = true +enabled = true # specify organization name that should be used for unauthenticated users org_name = Main Org. diff --git a/src/grafana/provisioning/dashboards/demo.yaml b/src/grafana/provisioning/dashboards/demo.yaml index 5c685e11d0..b75b4a478e 100644 --- a/src/grafana/provisioning/dashboards/demo.yaml +++ b/src/grafana/provisioning/dashboards/demo.yaml @@ -7,6 +7,7 @@ providers: - name: 'OpenTelemetry Demo' orgId: 1 folder: 'Demo' + folderUid: 'demo' type: file disableDeletion: false editable: true diff --git a/src/grafana/provisioning/dashboards/demo/demo-dashboard.json b/src/grafana/provisioning/dashboards/demo/demo-dashboard.json index 46814412e7..e86d5534f8 100644 --- a/src/grafana/provisioning/dashboards/demo/demo-dashboard.json +++ b/src/grafana/provisioning/dashboards/demo/demo-dashboard.json @@ -23,8 +23,8 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, + "graphTooltip": 1, + "id": 2, "links": [], "panels": [ { @@ -33,7 +33,7 @@ "overrides": [] }, "gridPos": { - "h": 2, + "h": 3, "w": 24, "x": 0, "y": 0 @@ -45,10 +45,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "This dashboard shows RED metrics for the selected service, as generated by the spanmetrics connector in the OpenTelemetry Collector.\nIf the selected service emits logs, the logs will also be displayed.\nCustom metrics generated by some services are also displayed.\n
\nChart panels may require 5 minutes after the Demo is started before rendering data.", + "content": "This dashboard shows RED metrics for the selected service, as generated by the spanmetrics connector in the OpenTelemetry Collector.\nIf the selected service emits logs, the logs will also be displayed.\nCustom metrics generated by some services are also displayed. \n
\nChart panels may require 5 minutes after the Demo is started before rendering data.", "mode": "html" }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "title": "", "type": "text" }, @@ -58,11 +58,11 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 3 }, "id": 14, "panels": [], - "title": "Spanmetrics", + "title": "Spanmetrics (RED metrics)", "type": "row" }, { @@ -70,7 +70,6 @@ "type": "prometheus", "uid": "webstore-metrics" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -99,7 +98,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -123,17 +122,17 @@ } ] }, - "unit": "dtdurationms" + "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 3 + "y": 4 }, - "id": 2, + "id": 12, "interval": "2m", "options": { "legend": { @@ -143,11 +142,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -155,53 +155,13 @@ "uid": "webstore-metrics" }, "editorMode": "code", - "exemplar": true, - "expr": "histogram_quantile(0.50, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))", - "legendFormat": "quantile50", + "expr": "sum by (span_name) (rate(traces_span_metrics_duration_milliseconds_count{service_name=\"${service}\"}[$__rate_interval]))", + "legendFormat": "{{ span_name }}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.95, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "quantile95", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "quantile99", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.999, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le))", - "hide": false, - "legendFormat": "quantile999", - "range": true, - "refId": "D" } ], - "title": "Latency for ${service}", + "title": "Requests Rate by Span Name", "type": "timeseries" }, { @@ -238,7 +198,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -267,9 +227,9 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 3 + "w": 8, + "x": 8, + "y": 4 }, "id": 10, "interval": "2m", @@ -281,11 +241,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -293,14 +254,14 @@ "uid": "webstore-metrics" }, "editorMode": "code", - "expr": " sum by (span_name) (rate(traces_span_metrics_calls_total{status_code=\"STATUS_CODE_ERROR\", service_name=\"${service}\"}[$__rate_interval]))", + "expr": "sum by (span_name) (rate(traces_span_metrics_calls_total{status_code=\"STATUS_CODE_ERROR\", service_name=\"${service}\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{ span_name }}", "range": true, "refId": "A" } ], - "title": "Error Rate for ${service} by span name", + "title": "Error Rate by Span Name", "type": "timeseries" }, { @@ -308,6 +269,7 @@ "type": "prometheus", "uid": "webstore-metrics" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -336,7 +298,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -360,31 +322,32 @@ } ] }, - "unit": "reqps" + "unit": "dtdurationms" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 11 + "w": 8, + "x": 16, + "y": 4 }, - "id": 12, + "id": 2, "interval": "2m", "options": { "legend": { "calcs": [], - "displayMode": "table", + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -392,13 +355,14 @@ "uid": "webstore-metrics" }, "editorMode": "code", - "expr": "sum by (span_name) (rate(traces_span_metrics_duration_milliseconds_count{service_name=\"${service}\"}[$__rate_interval]))", - "legendFormat": "{{ span_name }}", + "exemplar": false, + "expr": "histogram_quantile(0.50, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name=\"${service}\"}[$__rate_interval])) by (le, span_name))", + "legendFormat": "{{span_name}}", "range": true, "refId": "A" } ], - "title": "Requests Rate for ${service} by span name", + "title": "Average Duration by Span Name", "type": "timeseries" }, { @@ -407,17 +371,17 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 12 }, "id": 19, "panels": [], - "title": "Application Logs", + "title": "Application Log Records", "type": "row" }, { "datasource": { "type": "grafana-opensearch-datasource", - "uid": "P9744FCCEAAFBD98F" + "uid": "webstore-logs" }, "description": "", "fieldConfig": { @@ -466,7 +430,7 @@ "h": 8, "w": 4, "x": 0, - "y": 20 + "y": 13 }, "id": 20, "options": { @@ -481,23 +445,26 @@ }, "showHeader": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "alias": "", "bucketAggs": [ { - "field": "time", - "id": "2", + "field": "severity.text.keyword", + "id": "3", "settings": { - "interval": "auto" + "min_doc_count": "1", + "order": "desc", + "orderBy": "_count", + "size": "0" }, - "type": "date_histogram" + "type": "terms" } ], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "P9744FCCEAAFBD98F" + "uid": "webstore-logs" }, "format": "table", "metrics": [ @@ -509,16 +476,30 @@ "query": "search source=otel\n| where resource.service.name=\"${service}\"\n| stats count() by severity.text", "queryType": "PPL", "refId": "A", - "timeField": "time" + "timeField": "observedTimestamp" + } + ], + "title": "Log Records by Severity", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Count": "", + "severity.text.keyword": "Severity" + } + } } ], - "title": "${service} Log entries by Severity", "type": "table" }, { "datasource": { "type": "grafana-opensearch-datasource", - "uid": "P9744FCCEAAFBD98F" + "uid": "webstore-logs" }, "fieldConfig": { "defaults": { @@ -528,9 +509,11 @@ "custom": { "align": "auto", "cellOptions": { - "type": "auto" + "type": "auto", + "wrapText": false }, - "inspect": false + "filterable": true, + "inspect": true }, "mappings": [], "thresholds": { @@ -553,10 +536,29 @@ "id": "byName", "options": "observedTimestamp" }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "body" + }, "properties": [ { "id": "custom.width", - "value": 222 + "value": 386 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "severity.text" + }, + "properties": [ + { + "id": "custom.width", + "value": 127 } ] } @@ -566,7 +568,7 @@ "h": 8, "w": 20, "x": 4, - "y": 20 + "y": 13 }, "id": 17, "options": { @@ -580,46 +582,72 @@ "show": false }, "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "observedTimestamp" - } - ] + "sortBy": [] }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "alias": "", - "bucketAggs": [ - { - "field": "time", - "id": "2", - "settings": { - "interval": "auto" - }, - "type": "date_histogram" - } - ], + "bucketAggs": [], "datasource": { "type": "grafana-opensearch-datasource", - "uid": "P9744FCCEAAFBD98F" + "uid": "webstore-logs" }, - "format": "logs", + "format": "table", "hide": false, "metrics": [ { "id": "1", - "type": "count" + "settings": { + "order": "desc", + "size": "100", + "useTimeRange": true + }, + "type": "raw_data" } ], - "query": "search source=otel\n| where resource.service.name=\"${service}\"| sort - observedTimestamp | head 100", + "query": "search source=otel\n| where resource.service.name=\"${service}\"\n| sort - observedTimestamp \n| head 100", "queryType": "PPL", "refId": "A", "timeField": "observedTimestamp" } ], - "title": "${service} Logs (100 recent entries)", + "title": "Log Records (100 recent entries)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "@timestamp": true + }, + "includeByName": {}, + "indexByName": { + "@timestamp": 1, + "attributes.data_stream.dataset": 4, + "attributes.data_stream.namespace": 5, + "attributes.data_stream.type": 6, + "attributes.productId": 7, + "attributes.quantity": 8, + "attributes.userId": 9, + "body": 3, + "instrumentationScope.name": 10, + "observedTimestamp": 0, + "resource.container.id": 11, + "resource.docker.cli.cobra.command_path": 12, + "resource.host.name": 13, + "resource.service.name": 14, + "resource.telemetry.sdk.language": 15, + "resource.telemetry.sdk.name": 16, + "resource.telemetry.sdk.version": 17, + "severity.number": 18, + "severity.text": 2, + "spanId": 19, + "traceId": 20 + }, + "renameByName": {} + } + } + ], "type": "table" }, { @@ -628,7 +656,7 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 21 }, "id": 18, "panels": [], @@ -668,7 +696,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -683,7 +711,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -699,7 +728,7 @@ "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 22 }, "id": 6, "options": { @@ -710,11 +739,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -788,7 +818,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -803,7 +833,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -819,7 +850,7 @@ "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 22 }, "id": 8, "options": { @@ -830,11 +861,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -878,7 +910,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "bars", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -893,7 +925,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -908,7 +940,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -923,7 +956,7 @@ "h": 8, "w": 12, "x": 0, - "y": 37 + "y": 30 }, "id": 4, "options": { @@ -934,11 +967,12 @@ "showLegend": false }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -989,7 +1023,7 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -1004,7 +1038,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1019,7 +1054,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 30 }, "id": 16, "interval": "2m", @@ -1031,11 +1066,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", - "sort": "none" + "hideZeros": false, + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1052,6 +1088,59 @@ ], "title": "Quote Service batch span processor", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 23, + "panels": [], + "title": "Service Dependency", + "type": "row" + }, + { + "datasource": { + "type": "jaeger", + "uid": "webstore-traces" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 22, + "options": { + "edges": { + "mainStatUnit": "requests" + }, + "nodes": { + "arcs": [], + "mainStatUnit": "" + }, + "zoomMode": "cooperative" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "jaeger", + "uid": "webstore-traces" + }, + "queryType": "dependencyGraph", + "refId": "A" + } + ], + "title": "Service Dependency", + "type": "nodeGraph" } ], "preload": false, @@ -1071,6 +1160,7 @@ }, "definition": "traces_span_metrics_duration_milliseconds_bucket", "includeAll": false, + "label": "Service", "name": "service", "options": [], "query": { @@ -1078,14 +1168,14 @@ "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": "/.*service_name=\\\"([^\\\"]+)\\\".*/", + "regex": "/.*service.name=\\\"([^\\\"]+)\\\".*/", "sort": 1, "type": "query" } ] }, "time": { - "from": "now-1h", + "from": "now-15m", "to": "now" }, "timepicker": {}, diff --git a/src/grafana/provisioning/dashboards/demo/exemplars-dashboard.json b/src/grafana/provisioning/dashboards/demo/exemplars-dashboard.json index 6ac22bc780..9c3ec2611e 100644 --- a/src/grafana/provisioning/dashboards/demo/exemplars-dashboard.json +++ b/src/grafana/provisioning/dashboards/demo/exemplars-dashboard.json @@ -447,7 +447,7 @@ "list": [] }, "time": { - "from": "now-1h", + "from": "now-15m", "to": "now" }, "timepicker": {}, diff --git a/src/grafana/provisioning/dashboards/demo/opentelemetry-collector-data-flow.json b/src/grafana/provisioning/dashboards/demo/opentelemetry-collector-data-flow.json deleted file mode 100644 index fbcc7df945..0000000000 --- a/src/grafana/provisioning/dashboards/demo/opentelemetry-collector-data-flow.json +++ /dev/null @@ -1,2332 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "otelcol metrics dashboard", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 6, - "links": [], - "panels": [ - { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 56, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "This dashboard is used to understand the data flows for the OpenTelemetry Collector's metrics and traces pipelines.\n
\nChart panels may require 5 minutes after the Demo is started before rendering data.", - "mode": "html" - }, - "pluginVersion": "11.4.0", - "title": "", - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 8, - "panels": [], - "title": "Process", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Otel Collector Instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 3 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "count(count(otelcol_process_cpu_seconds_total{service_instance_id=~\".*\"}) by (service_instance_id))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Instance", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "otelcol_process_cpu_seconds", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 3 - }, - "id": 24, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "avg(rate(otelcol_process_cpu_seconds_total{}[$__rate_interval])*100) by (instance)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Cpu", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Memory Rss\navg(otelcol_process_memory_rss{}) by (instance)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 3 - }, - "id": 38, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "avg(otelcol_process_memory_rss{}) by (instance)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Memory", - "type": "gauge" - }, - { - "description": "", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 15, - "x": 9, - "y": 3 - }, - "id": 32, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "## Opentelemetry Collector Data Ingress/Egress\n\n`service_version:` ${service_version}\n\n`opentelemetry collector:` contrib\n\n", - "mode": "markdown" - }, - "pluginVersion": "11.4.0", - "title": "", - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 10, - "panels": [], - "title": "Trace Pipeline", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "(avg(sum by(job) (rate(otelcol_exporter_sent_spans{}[$__range]))) / avg(sum by(job) (rate(otelcol_receiver_accepted_spans{}[$__range])))) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-blue", - "value": null - }, - { - "color": "semi-dark-red", - "value": 0 - }, - { - "color": "super-light-orange", - "value": 0.4 - }, - { - "color": "dark-blue", - "value": 0.9 - }, - { - "color": "super-light-orange", - "value": 1.2 - }, - { - "color": "dark-red", - "value": 2.1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 19, - "w": 3, - "x": 0, - "y": 8 - }, - "id": 55, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(sum by(job) (rate(otelcol_exporter_sent_spans_total{}[$__range])))", - "format": "time_series", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "export" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "avg(sum by(job) (rate(otelcol_receiver_accepted_spans_total{}[$__range])))", - "format": "time_series", - "hide": true, - "legendFormat": "__auto", - "range": true, - "refId": "acc" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "(avg(sum by(job) (rate(otelcol_exporter_sent_spans_total{}[$__range]))) / avg(sum by(job) (rate(otelcol_receiver_accepted_spans_total{}[$__range])))) ", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Export Ratio", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 21, - "x": 3, - "y": 8 - }, - "id": 4, - "options": { - "edges": {}, - "nodes": { - "mainStatUnit": "flops" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_join(label_join(\n(rate(otelcol_receiver_accepted_spans_total{}[$__interval]))\n, \"id\", \"\", \"transport\", \"receiver\")\n, \"title\", \"\", \"transport\", \"receiver\")\n\nor\n\nlabel_replace(label_replace(\nsum by(service_name) (rate(otelcol_receiver_accepted_spans_total{}[$__interval]))\n, \"id\", \"processor\", \"dummynode\", \"\")\n, \"title\", \"processor\", \"dummynode\", \"\")\n\nor\nlabel_replace(label_replace(\n(rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n, \"id\", \"$0\", \"processor\", \".*\")\n, \"title\", \"$0\", \"processor\", \".*\")\n\nor\nlabel_replace(label_replace(\nsum by(exporter) (rate(otelcol_exporter_sent_spans_total{}[$__interval]))\n, \"id\", \"exporter\", \"dummynode\", \"\")\n, \"title\", \"exporter\", \"dummynode\", \"\")\n \nor\nlabel_replace(label_replace(\nsum by(exporter) (rate(otelcol_exporter_sent_spans_total{}[$__interval]))\n, \"id\", \"$0\", \"exporter\", \".*\")\n, \"title\", \"$0\", \"exporter\", \".*\")", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_join(\nlabel_replace(label_join(\n(rate(otelcol_receiver_accepted_spans_total{}[$__interval]))\n\n ,\"source\",\"\",\"transport\",\"receiver\")\n,\"target\",\"processor\",\"\",\"\")\n,\"id\",\"-\",\"source\",\"target\")\n\n or\n\n label_join(\nlabel_replace(label_replace(\n (rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n ,\"source\",\"processor\",\"\",\"\")\n,\"target\",\"$0\",\"processor\",\".*\")\n,\"id\",\"-\",\"source\",\"target\")\n\nor\n label_join(\nlabel_replace(label_replace(\n (rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n ,\"source\",\"$0\",\"processor\",\".*\")\n,\"target\",\"exporter\",\"\",\"\")\n,\"id\",\"-\",\"source\",\"target\")\n\nor\n label_join(\nlabel_replace(label_replace(\n (rate(otelcol_exporter_sent_spans_total{}[$__interval]))\n ,\"source\",\"exporter\",\"\",\"\")\n,\"target\",\"$0\",\"exporter\",\".*\")\n,\"id\",\"-\",\"source\",\"target\")\n\n", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "", - "type": "nodeGraph" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Spans Accepted by Receiver and Transport", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "noValue": "no data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 3, - "y": 19 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_accepted_spans_total{}[$__rate_interval])) by (receiver,transport)", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Accepted", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Spans Accepted ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 8, - "y": 19 - }, - "id": 13, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_accepted_spans_total{}[$__rate_interval])) ", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Batch Processed", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 11, - "y": 19 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_processor_batch_batch_send_size_sum{}[$__rate_interval])) by (processor)", - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Batch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 16, - "y": 19 - }, - "id": 14, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_sent_spans_total{}[$__interval])) ", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Sent by Exporter", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 19, - "y": 19 - }, - "id": 30, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_sent_spans_total{}[$__rate_interval])) by (exporter)", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Sent", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "noValue": "no data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 3, - "y": 24 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_refused_spans_total{}[$__rate_interval])) by (receiver,transport)", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Refused", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Spans Accepted ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 8, - "y": 24 - }, - "id": 18, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_refused_spans_total{}[$__rate_interval])) ", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "otelcol_exporter_send_failed_spans", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 16, - "y": 24 - }, - "id": 19, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_send_failed_spans_total{}[$__rate_interval])) ", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Sent by Exporter\notelcol_exporter_send_failed_spans", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 19, - "y": 24 - }, - "id": 20, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_send_failed_spans_total{}[$__rate_interval])) by (exporter)", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Failed", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 22, - "panels": [], - "title": "Metrics Pipeline", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "avg(sum by(job) (rate(otelcol_exporter_sent_metric_points{}[$__range]))) versus avg(sum by(job) (rate(otelcol_receiver_accepted_metric_points{}[$__range])))", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-blue" - }, - { - "color": "semi-dark-red", - "value": 0 - }, - { - "color": "super-light-orange", - "value": 0.4 - }, - { - "color": "dark-blue", - "value": 0.9 - }, - { - "color": "super-light-orange", - "value": 1.2 - }, - { - "color": "dark-red", - "value": 2.1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 19, - "w": 3, - "x": 0, - "y": 28 - }, - "id": 54, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(sum by(job) (rate(otelcol_exporter_sent_metric_points_total{}[$__range])))", - "format": "time_series", - "hide": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "export" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "avg(sum by(job) (rate(otelcol_receiver_accepted_metric_points_total{}[$__range])))", - "format": "time_series", - "hide": true, - "legendFormat": "__auto", - "range": true, - "refId": "acc" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "( avg(sum by(job) (rate(otelcol_exporter_sent_metric_points_total{}[$__range]))) /avg(sum by(job) (rate(otelcol_receiver_accepted_metric_points_total{}[$__range]))))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Export Ratio", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "percent", - "binary": { - "left": "avg(sum by(job) (rate(otelcol_exporter_sent_metric_points{}[3600s])))", - "operator": "/", - "reducer": "sum", - "right": "avg(sum by(job) (rate(otelcol_receiver_accepted_metric_points{}[3600s])))" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "(sum(rate(otelcol_exporter_sent_metric_points{exporter=\"prometheus\"}[1m0s])) )": true, - "Time": true, - "avg(sum by(job) (rate(otelcol_exporter_sent_metric_points{}[3600s])))": true, - "avg(sum by(job) (rate(otelcol_receiver_accepted_metric_points{}[3600s])))": true, - "{instance=\"otelcol:9464\", job=\"otel\"}": true - }, - "indexByName": {}, - "renameByName": { - "Time": "", - "percent": "Percent", - "{exporter=\"debug\", instance=\"otelcol:8888\", job=\"otel-collector\", service_instance_id=\"fbfa720a-ebf9-45c8-a79a-9d3b6021a663\", service_name=\"otelcol-contrib\", service_version=\"0.70.0\"}": "" - } - } - } - ], - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Metrics Signalling Pipelines", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 21, - "x": 3, - "y": 28 - }, - "id": 25, - "options": { - "edges": {}, - "nodes": { - "mainStatUnit": "flops" - } - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "\nlabel_join(label_join(\n(rate(otelcol_receiver_accepted_metric_points_total{}[$__interval]))\n, \"id\", \"\", \"transport\", \"receiver\")\n, \"title\", \"\", \"transport\", \"receiver\")\n\nor\n\nlabel_replace(label_replace(\nsum by(service_name) (rate(otelcol_receiver_accepted_spans_total{}[$__interval]))\n, \"id\", \"processor\", \"dummynode\", \"\")\n, \"title\", \"processor\", \"dummynode\", \"\")\n\n\n\nor\nlabel_replace(label_replace(\n(rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n, \"id\", \"$0\", \"processor\", \".*\")\n, \"title\", \"$0\", \"processor\", \".*\")\n\n\n\n\n\nor\nlabel_replace(label_replace(\nsum (rate(otelcol_exporter_sent_metric_points_total{}[$__interval]))\n, \"id\", \"exporter\", \"dummynode\", \"\")\n, \"title\", \"exporter\", \"dummynode\", \"\")\n\nor\nlabel_replace(label_replace(\nsum by(exporter) (rate(otelcol_exporter_sent_metric_points_total{}[$__interval]))\n, \"id\", \"$0\", \"exporter\", \".*\")\n, \"title\", \"$0\", \"exporter\", \".*\")", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_join(\nlabel_replace(label_join(\n(rate(otelcol_receiver_accepted_metric_points_total{}[$__interval]))\n\n,\"source\",\"\",\"transport\",\"receiver\")\n,\"target\",\"processor\",\"\",\"\")\n,\"id\",\"-\",\"source\",\"target\")\n\n\nor\n\nlabel_join(\nlabel_replace(label_replace(\n(rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n,\"source\",\"processor\",\"\",\"\")\n,\"target\",\"$0\",\"processor\",\".*\")\n,\"id\",\"-\",\"source\",\"target\")\n\n\n\n\n\nor\n\n\nlabel_join(\nlabel_replace(label_replace(\n(rate(otelcol_processor_batch_batch_send_size_count{}[$__interval]))\n,\"source\",\"$0\",\"processor\",\".*\")\n,\"target\",\"exporter\",\"\",\"\")\n,\"id\",\"-\",\"source\",\"target\")\n\nor\nlabel_join(\nlabel_replace(label_replace(\n(rate(otelcol_exporter_sent_metric_points_total{}[$__interval]))\n,\"source\",\"exporter\",\"\",\"\")\n,\"target\",\"$0\",\"exporter\",\".*\")\n,\"id\",\"-\",\"source\",\"target\")", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "", - "type": "nodeGraph" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "otelcol_receiver_accepted_metric_points", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "noValue": "no data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 3, - "y": 39 - }, - "id": 26, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{}[$__rate_interval])) by (receiver,transport)", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Accepted", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "otelcol_receiver_accepted_metric_points\nTotal Accepted ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 8, - "y": 39 - }, - "id": 27, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{}[$__rate_interval])) ", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "#EAB839", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 11, - "y": 39 - }, - "id": 28, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_processor_batch_batch_send_size_sum{}[$__rate_interval])) by (processor)", - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Batch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Export ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 16, - "y": 39 - }, - "id": 29, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{}[$__rate_interval])) ", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Sent by Exporter", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 19, - "y": 39 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{}[$__rate_interval])) by (exporter) ", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Sent", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "noValue": "no data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 3, - "y": 44 - }, - "id": 47, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_refused_metric_points_total{}[$__rate_interval])) by (receiver,transport)", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Refused", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Refused \nsum(rate(otelcol_receiver_refused_metric_points{}[$__rate_interval])) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 8, - "y": 44 - }, - "id": 48, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "sum(rate(otelcol_receiver_refused_metric_points_total{}[$__rate_interval])) ", - "legendFormat": "{{receiver}}-{{transport}}", - "range": true, - "refId": "A" - } - ], - "title": "Total ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Total Failed Export ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 16, - "y": 44 - }, - "id": 49, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total{}[$__rate_interval])) ", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Total", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "Sent by Exporter\notelcol_exporter_send_failed_metric_points", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 19, - "y": 44 - }, - "id": 50, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total{}[$__rate_interval])) by (exporter)", - "format": "time_series", - "instant": false, - "legendFormat": "{{processor}}", - "range": true, - "refId": "A" - } - ], - "title": "Failed", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 35, - "panels": [], - "title": "Prometheus Scrape", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "otelcol prometheus exporter 8888 export rate versus prometheus scrape metrics", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-blue" - }, - { - "color": "semi-dark-red", - "value": 0 - }, - { - "color": "super-light-orange", - "value": 0.4 - }, - { - "color": "dark-blue", - "value": 0.9 - }, - { - "color": "super-light-orange", - "value": 1.2 - }, - { - "color": "dark-red", - "value": 2.1 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 0, - "y": 48 - }, - "id": 53, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": false, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(sum_over_time(scrape_samples_scraped{job=\"otel-collector\"}[$__range])/ count_over_time(scrape_samples_scraped{job=\"otel-collector\"}[$__range])/(5*30)) ", - "format": "time_series", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "accepted" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "expr": "(sum(rate(otelcol_exporter_sent_metric_points_total{exporter=\"prometheus\"}[$__rate_interval])) )", - "format": "time_series", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Exported/Scraped", - "transformations": [ - { - "id": "calculateField", - "options": { - "alias": "percent", - "binary": { - "left": "{instance=\"otelcol:9464\", job=\"otel-collector\"}", - "operator": "/", - "reducer": "sum", - "right": "(sum(rate(otelcol_exporter_sent_metric_points{exporter=\"prometheus\"}[1m0s])) )" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "(sum(rate(otelcol_exporter_sent_metric_points{exporter=\"prometheus\"}[1m0s])) )": true, - "Time": true, - "{instance=\"otelcol:9464\", job=\"otel\"}": true - }, - "indexByName": {}, - "renameByName": { - "percent": "Percent" - } - } - } - ], - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "sum_over_time(scrape_samples_scraped[$__range])/ count_over_time(scrape_samples_scraped[$__range])", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 5, - "x": 3, - "y": 48 - }, - "id": 37, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum_over_time(scrape_samples_scraped[$__range])/ count_over_time(scrape_samples_scraped[$__range])/(5*30)", - "format": "time_series", - "instant": false, - "legendFormat": "{{job}}/{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Samples Scraped", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "description": "scrape_samples_scraped{job!=\"\"}\nTotal Samples Scraped", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 8, - "y": 48 - }, - "id": 42, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum_over_time(scrape_samples_scraped[$__range])/ count_over_time(scrape_samples_scraped[$__range])/(5*30)", - "format": "time_series", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Total", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "include": [ - "{instance=\"otelcol:8888\", job=\"otel-collector\"}" - ], - "reducer": "sum" - }, - "replaceFields": true - } - } - ], - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 11, - "y": 48 - }, - "id": 41, - "options": { - "edges": {}, - "nodes": {} - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(label_replace(label_replace(\nsum (scrape_samples_scraped{job!=\"\"}) by (instance)\n, \"id\", \"$0\", \"instance\", \".*\")\n, \"title\", \"$0\", \"instance\", \".*\")\n,\"mainstat\",\"\",\"\",\"\")\n\nor \n\nlabel_replace(label_replace(label_replace(\nsum (scrape_samples_scraped{job!=\"\"})\n, \"id\", \"prometheus\", \"\", \"\")\n, \"title\", \"prometheus\", \"\", \"\")\n,\"mainstat\",\"\",\"\",\"\")\n", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" - }, - { - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_join(\nlabel_replace(label_replace(\nsum (scrape_samples_scraped{job!=\"\"}) by (instance)\n,\"source\",\"$0\",\"instance\",\".*\")\n,\"target\",\"prometheus\",\"\",\"\")\n,\"id\",\"-\",\"source\",\"target\")", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true - }, - "indexByName": {}, - "renameByName": {} - } - } - ], - "type": "nodeGraph" - }, - { - "description": "Sent by Exporter", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 5, - "x": 19, - "y": 48 - }, - "id": 52, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "\n \n## Prometheus Config\n\n`evaluation_interval:` 30s\n\n`scrape_interval:` 5s", - "mode": "markdown" - }, - "pluginVersion": "11.3.0", - "title": "", - "type": "text" - } - ], - "preload": false, - "refresh": false, - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "allValue": ".*", - "current": { - "text": "0.70.0", - "value": "0.70.0" - }, - "datasource": { - "type": "prometheus", - "uid": "webstore-metrics" - }, - "definition": "query_result(sum(otelcol_process_uptime{}) by (service_version))\n", - "hide": 2, - "includeAll": false, - "label": "service_version", - "multi": true, - "name": "service_version", - "options": [], - "query": { - "query": "query_result(sum(otelcol_process_uptime{}) by (service_version))\n", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/.*service_version=\"(.*)\".*/", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Opentelemetry Collector Data Flow", - "uid": "rl5_tea4k", - "version": 2, - "weekStart": "" -} \ No newline at end of file diff --git a/src/grafana/provisioning/dashboards/demo/opentelemetry-collector.json b/src/grafana/provisioning/dashboards/demo/opentelemetry-collector.json index 529c269785..f867efa117 100644 --- a/src/grafana/provisioning/dashboards/demo/opentelemetry-collector.json +++ b/src/grafana/provisioning/dashboards/demo/opentelemetry-collector.json @@ -21,11 +21,11 @@ } ] }, - "description": "Visualize OpenTelemetry (OTEL) collector metrics (tested with OTEL contrib v0.113.0)", + "description": "Visualize OpenTelemetry (OTEL) collector metrics (tested with OTEL contrib v0.120.1)", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 2, + "id": 4, "links": [], "panels": [ { @@ -39,17 +39,17 @@ "x": 0, "y": 0 }, - "id": 62, + "id": 86, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, - "content": "This dashboard uses the metrics generated by the OpenTelemetry Collector.\nIt is used to understand the overall performance and health of the OpenTelemetry Collector.\n
\nChart panels may require 5 minutes after the Demo is started before rendering data.", - "mode": "markdown" + "content": "This dashboard uses the metrics generated by the OpenTelemetry Collector.\nIt is used to understand the overall performance and health of the OpenTelemetry Collector.\n
\nChart panels may require 5 minutes after the Demo is started before rendering data.\n", + "mode": "html" }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "title": "", "type": "text" }, @@ -141,6 +141,10 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -166,11 +170,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -178,8 +183,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_accepted_spans_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_spans${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -193,8 +198,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_refused_spans_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_refused_spans${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -282,6 +287,10 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -293,7 +302,7 @@ "x": 8, "y": 3 }, - "id": 32, + "id": 80, "interval": "$minstep", "options": { "legend": { @@ -307,11 +316,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -319,8 +329,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_accepted_metric_points_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_metric_points${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -334,8 +344,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_refused_metric_points_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_refused_metric_points${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -423,6 +433,10 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -448,11 +462,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -460,8 +475,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_accepted_log_records_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_log_records${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, @@ -475,8 +490,8 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_receiver_refused_log_records_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_receiver_refused_log_records${suffix_total}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", @@ -548,7 +563,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -568,7 +582,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Refused.*/" + "options": "/Refused.*/" }, "properties": [ { @@ -577,13 +591,17 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*Dropped.*/" + "options": "/Dropped.*/" }, "properties": [ { @@ -592,6 +610,10 @@ "fixedColor": "purple", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -603,7 +625,7 @@ "x": 0, "y": 12 }, - "id": 36, + "id": 85, "interval": "$minstep", "options": { "legend": { @@ -617,11 +639,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -629,129 +652,41 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_count{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "$minstep", - "intervalFactor": 1, - "legendFormat": "Batch send size count: {{processor}}", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_sum{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_incoming_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"traces\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", - "hide": false, - "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Batch send size sum: {{processor}}", + "legendFormat": "Incomming: {{processor}} {{service_instance_id}}", + "range": true, "refId": "A" - } - ], - "title": "Batch Metrics", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Number of units in the batch", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - }, - "links": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 49, - "interval": "$minstep", - "maxDataPoints": 50, - "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": true, - "scale": "exponential", - "scheme": "Reds", - "steps": 57 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false - } - }, - "pluginVersion": "11.4.0", - "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(otelcol_processor_batch_batch_send_size_bucket{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (le)", - "format": "heatmap", + "exemplar": false, + "expr": "0-sum(${metric:value}(otelcol_processor_outgoing_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"traces\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", "hide": false, - "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "{{le}}", + "legendFormat": "Outgoing: {{processor}} {{service_instance_id}}", + "range": true, "refId": "B" } ], - "title": "Batch Send Size Heatmap", - "type": "heatmap" + "title": "Spans ${metric:text}", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "description": "Number of times the batch was sent due to a size trigger. Number of times the batch was sent due to a timeout trigger.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -792,7 +727,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -812,7 +746,7 @@ { "matcher": { "id": "byRegexp", - "options": "/.*Refused.*/" + "options": "/Refused.*/" }, "properties": [ { @@ -821,13 +755,17 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] }, { "matcher": { "id": "byRegexp", - "options": "/.*Dropped.*/" + "options": "/Dropped.*/" }, "properties": [ { @@ -836,6 +774,10 @@ "fixedColor": "purple", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -844,10 +786,10 @@ "gridPos": { "h": 8, "w": 8, - "x": 16, + "x": 8, "y": 12 }, - "id": 56, + "id": 83, "interval": "$minstep", "options": { "legend": { @@ -861,11 +803,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -873,15 +816,14 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send_total {processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_incoming_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"metrics\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", - "hide": false, - "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Batch sent due to a timeout trigger: {{processor}}", - "refId": "B" + "legendFormat": "Incomming: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "A" }, { "datasource": { @@ -889,39 +831,26 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send{processor=~\"$processor\"}[$__rate_interval])) by (processor)", + "exemplar": false, + "expr": "0-sum(${metric:value}(otelcol_processor_outgoing_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"metrics\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, - "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Batch sent due to a timeout trigger: {{processor}}", - "refId": "A" + "legendFormat": "Outgoing: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "B" } ], - "title": "Batch Metrics", + "title": "Metric Points ${metric:text}", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 25, - "panels": [], - "title": "Exporters", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "description": "Sent: count/rate of spans successfully sent to destination.\nEngueue: count/rate of spans failed to be added to the sending queue.", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -962,7 +891,6 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -982,7 +910,7 @@ { "matcher": { "id": "byRegexp", - "options": "/Failed:.*/" + "options": "/Refused.*/" }, "properties": [ { @@ -991,18 +919,41 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } ] }, "gridPos": { - "h": 9, + "h": 8, "w": 8, - "x": 0, - "y": 21 + "x": 16, + "y": 12 }, - "id": 37, + "id": 84, "interval": "$minstep", "options": { "legend": { @@ -1016,11 +967,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", - "sort": "none" + "sort": "desc" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1028,12 +980,12 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_sent_spans_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_incoming_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"logs\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "legendFormat": "Incomming: {{processor}} {{service_instance_id}}", "range": true, "refId": "A" }, @@ -1043,34 +995,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_spans{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "0-sum(${metric:value}(otelcol_processor_outgoing_items${suffix_total}{processor=~\"$processor\",job=\"$job\",otel_signal=\"logs\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "legendFormat": "Outgoing: {{processor}} {{service_instance_id}}", "range": true, "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_send_failed_spans{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", - "format": "time_series", - "hide": false, - "interval": "$minstep", - "intervalFactor": 1, - "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", - "range": true, - "refId": "C" } ], - "title": "Spans ${metric:text}", + "title": "Logs Records ${metric:text}", "type": "timeseries" }, { @@ -1078,7 +1014,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Sent: count/rate of metric points successfully sent to destination.\nEngueue: count/rate of metric points failed to be added to the sending queue.", + "description": "Accepted: count/rate of spans successfully pushed into the next component in the pipeline.\nRefused: count/rate of spans that were rejected by the next component in the pipeline.\nDropped: count/rate of spans that were dropped", "fieldConfig": { "defaults": { "color": { @@ -1139,7 +1075,7 @@ { "matcher": { "id": "byRegexp", - "options": "/Failed:.*/" + "options": "/Refused.*/" }, "properties": [ { @@ -1148,18 +1084,41 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } ] }, "gridPos": { - "h": 9, + "h": 8, "w": 8, - "x": 8, - "y": 21 + "x": 0, + "y": 20 }, - "id": 38, + "id": 35, "interval": "$minstep", "options": { "legend": { @@ -1173,11 +1132,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1185,12 +1145,12 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_sent_metric_points_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_accepted_spans${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", "range": true, "refId": "A" }, @@ -1200,13 +1160,13 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_metric_points{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_refused_spans${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", "range": true, "refId": "B" }, @@ -1216,18 +1176,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_send_failed_metric_points{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_dropped_spans${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", "range": true, "refId": "C" } ], - "title": "Metric Points ${metric:text}", + "title": "Accepted Spans ${metric:text}", "type": "timeseries" }, { @@ -1235,7 +1195,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Sent: count/rate of log records successfully sent to destination.\nEngueue: count/rate of log records failed to be added to the sending queue.", + "description": "Accepted: count/rate of metric points successfully pushed into the next component in the pipeline.\nRefused: count/rate of metric points that were rejected by the next component in the pipeline.\nDropped: count/rate of metric points that were dropped", "fieldConfig": { "defaults": { "color": { @@ -1296,7 +1256,7 @@ { "matcher": { "id": "byRegexp", - "options": "/Failed:.*/" + "options": "/Refused.*/" }, "properties": [ { @@ -1305,18 +1265,41 @@ "fixedColor": "red", "mode": "fixed" } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" } ] } ] }, "gridPos": { - "h": 9, + "h": 8, "w": 8, - "x": 16, - "y": 21 + "x": 8, + "y": 20 }, - "id": 48, + "id": 50, "interval": "$minstep", "options": { "legend": { @@ -1330,11 +1313,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1342,12 +1326,12 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_sent_log_records_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_accepted_metric_points${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", "range": true, "refId": "A" }, @@ -1357,13 +1341,13 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_log_records{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_refused_metric_points${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", "range": true, "refId": "B" }, @@ -1373,18 +1357,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(${metric:value}(otelcol_exporter_send_failed_log_records{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_dropped_metric_points${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", "range": true, "refId": "C" } ], - "title": "Log Records ${metric:text}", + "title": "Accepted Metric Points ${metric:text}", "type": "timeseries" }, { @@ -1392,7 +1376,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Current size of the retry queue (in batches)", + "description": "Accepted: count/rate of log records successfully pushed into the next component in the pipeline.\nRefused: count/rate of log records that were rejected by the next component in the pipeline.\nDropped: count/rate of log records that were dropped", "fieldConfig": { "defaults": { "color": { @@ -1433,11 +1417,13 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1447,15 +1433,55 @@ }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 30 + "h": 8, + "w": 8, + "x": 16, + "y": 20 }, - "id": 10, + "id": 51, + "interval": "$minstep", "options": { "legend": { "calcs": [ @@ -1468,11 +1494,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1480,17 +1507,49 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(otelcol_exporter_queue_size{exporter=~\"$exporter\",job=\"$job\"}) by (exporter)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_accepted_log_records${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Max queue size: {{exporter}}", + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_refused_log_records${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_dropped_log_records${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "C" } ], - "title": "Exporter Queue Size", + "title": "Accepted Log Records ${metric:text}", "type": "timeseries" }, { @@ -1498,7 +1557,94 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Fixed capacity of the retry queue (in batches)", + "description": "Number of units in the batch", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 49, + "interval": "$minstep", + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 57 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(otelcol_processor_batch_batch_send_size_bucket{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "Batch Send Size Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1539,11 +1685,13 @@ }, "links": [], "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1553,15 +1701,29 @@ }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*count.*/" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 30 + "h": 8, + "w": 8, + "x": 8, + "y": 28 }, - "id": 55, + "id": 36, + "interval": "$minstep", "options": { "legend": { "calcs": [ @@ -1574,11 +1736,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1586,38 +1749,42 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "min(otelcol_exporter_queue_capacity{exporter=~\"$exporter\",job=\"$job\"}) by (exporter)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_count{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", + "hide": false, + "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Queue capacity: {{exporter}}", - "range": true, + "legendFormat": "Batch send size count: {{processor}} {{service_instance_id}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_sum{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Batch send size sum: {{processor}} {{service_instance_id}}", "refId": "A" } ], - "title": "Exporter Queue Capacity", + "title": "Batch Metrics 1", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 21, - "panels": [], - "title": "Collector", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "description": "Total physical memory (resident set size)", + "description": "Number of times the batch was sent due to a size trigger. Number of times the batch was sent due to a timeout trigger.", "fieldConfig": { "defaults": { "color": { @@ -1663,7 +1830,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1671,87 +1839,30 @@ } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Max Memory RSS " + "id": "byRegexp", + "options": "/.*timeout.*/" }, "properties": [ { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Avg Memory RSS " - }, - { - "id": "custom.lineWidth", - "value": 0 - }, - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Min Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Min Memory RSS " - }, - { - "id": "custom.fillOpacity", - "value": 20 + "id": "custom.axisPlacement", + "value": "right" } ] } ] }, "gridPos": { - "h": 9, + "h": 8, "w": 8, - "x": 0, - "y": 40 + "x": 16, + "y": 28 }, - "id": 40, + "id": 56, "interval": "$minstep", "options": { "legend": { @@ -1765,11 +1876,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1777,30 +1889,15 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_size_trigger_send${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, + "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Max Memory RSS {{service_instance_id}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "avg(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", - "format": "time_series", - "interval": "$minstep", - "intervalFactor": 1, - "legendFormat": "Avg Memory RSS {{service_instance_id}}", - "range": true, - "refId": "A" + "legendFormat": "Batch sent due to a size trigger: {{processor}}", + "refId": "B" }, { "datasource": { @@ -1808,26 +1905,39 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "min(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send${suffix_total}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", "format": "time_series", "hide": false, + "instant": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Min Memory RSS {{service_instance_id}}", - "range": true, - "refId": "B" + "legendFormat": "Batch sent due to a timeout trigger: {{processor}} {{service_instance_id}}", + "refId": "A" } ], - "title": "Total RSS Memory", + "title": "Batch Metrics 2", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 25, + "panels": [], + "title": "Exporters", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "description": "Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')", + "description": "Sent: count/rate of spans successfully sent to destination.\nEnqueue: count/rate of spans failed to be added to the sending queue.\nFailed: count/rate of spans in failed attempts to send to destination.", "fieldConfig": { "defaults": { "color": { @@ -1873,7 +1983,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1881,13 +1992,13 @@ } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Max Memory RSS " + "id": "byRegexp", + "options": "/Failed:.*/" }, "properties": [ { @@ -1898,58 +2009,8 @@ } }, { - "id": "custom.fillBelowTo", - "value": "Avg Memory RSS " - }, - { - "id": "custom.lineWidth", - "value": 0 - }, - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Min Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Min Memory RSS " - }, - { - "id": "custom.fillOpacity", - "value": 20 + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -1958,10 +2019,10 @@ "gridPos": { "h": 9, "w": 8, - "x": 8, - "y": 40 + "x": 0, + "y": 37 }, - "id": 52, + "id": 37, "interval": "$minstep", "options": { "legend": { @@ -1975,11 +2036,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -1987,15 +2049,14 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_sent_spans${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", - "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Max Memory RSS {{service_instance_id}}", + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "C" + "refId": "A" }, { "datasource": { @@ -2003,14 +2064,15 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "avg(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_spans${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", + "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "A" + "refId": "B" }, { "datasource": { @@ -2018,18 +2080,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "min(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_spans${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Min Memory RSS {{service_instance_id}}", + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "B" + "refId": "C" } ], - "title": "Total Runtime Sys Memory", + "title": "Spans ${metric:text}", "type": "timeseries" }, { @@ -2037,7 +2099,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')", + "description": "Sent: count/rate of metric points successfully sent to destination.\nEnqueue: count/rate of metric points failed to be added to the sending queue.\nFailed: count/rate of metric points in failed attempts to send to destination.", "fieldConfig": { "defaults": { "color": { @@ -2083,7 +2145,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2091,13 +2154,13 @@ } ] }, - "unit": "bytes" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Max Memory RSS " + "id": "byRegexp", + "options": "/Failed:.*/" }, "properties": [ { @@ -2108,58 +2171,8 @@ } }, { - "id": "custom.fillBelowTo", - "value": "Avg Memory RSS " - }, - { - "id": "custom.lineWidth", - "value": 0 - }, - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Min Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg Memory RSS " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Min Memory RSS " - }, - { - "id": "custom.fillOpacity", - "value": 20 + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -2168,10 +2181,10 @@ "gridPos": { "h": 9, "w": 8, - "x": 16, - "y": 40 + "x": 8, + "y": 37 }, - "id": 53, + "id": 38, "interval": "$minstep", "options": { "legend": { @@ -2185,11 +2198,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2197,15 +2211,14 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_sent_metric_points${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", - "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Max Memory RSS {{service_instance_id}}", + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "C" + "refId": "A" }, { "datasource": { @@ -2213,14 +2226,15 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "avg(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_metric_points${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", + "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "A" + "refId": "B" }, { "datasource": { @@ -2228,18 +2242,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "min(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_metric_points${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Min Memory RSS {{service_instance_id}}", + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "B" + "refId": "C" } ], - "title": "Total Runtime Heap Memory", + "title": "Metric Points ${metric:text}", "type": "timeseries" }, { @@ -2247,7 +2261,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Total CPU user and system time in percentage", + "description": "Sent: count/rate of log records successfully sent to destination.\nEnqueue: count/rate of log records failed to be added to the sending queue.\nFailed: count/rate of log records in failed attempts to send to destination.", "fieldConfig": { "defaults": { "color": { @@ -2293,7 +2307,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2301,13 +2316,13 @@ } ] }, - "unit": "percent" + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Max CPU usage " + "id": "byRegexp", + "options": "/Failed:.*/" }, "properties": [ { @@ -2318,54 +2333,8 @@ } }, { - "id": "custom.fillBelowTo", - "value": "Avg CPU usage " - }, - { - "id": "custom.lineWidth", - "value": 0 - }, - { - "id": "custom.fillOpacity", - "value": 20 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Avg CPU usage " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - { - "id": "custom.fillBelowTo", - "value": "Min CPU usage " - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Min CPU usage " - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - }, - { - "id": "custom.lineWidth", - "value": 0 + "id": "custom.axisPlacement", + "value": "right" } ] } @@ -2374,10 +2343,10 @@ "gridPos": { "h": 9, "w": 8, - "x": 0, - "y": 49 + "x": 16, + "y": 37 }, - "id": 39, + "id": 48, "interval": "$minstep", "options": { "legend": { @@ -2391,11 +2360,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2403,15 +2373,14 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(rate(otelcol_process_cpu_seconds_total{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_sent_log_records${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", - "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Max CPU usage {{service_instance_id}}", + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "B" + "refId": "A" }, { "datasource": { @@ -2419,15 +2388,15 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "avg(rate(otelcol_process_cpu_seconds_total{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_log_records${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Avg CPU usage {{service_instance_id}}", + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "A" + "refId": "B" }, { "datasource": { @@ -2435,18 +2404,18 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "min(rate(otelcol_process_cpu_seconds_total{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_log_records${suffix_total}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", "format": "time_series", "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Min CPU usage {{service_instance_id}}", + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", "range": true, "refId": "C" } ], - "title": "CPU Usage", + "title": "Log Records ${metric:text}", "type": "timeseries" }, { @@ -2454,7 +2423,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Number of service instances, which are reporting metrics", + "description": "Current size of the retry queue (in batches)", "fieldConfig": { "defaults": { "color": { @@ -2495,12 +2464,12 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2515,11 +2484,10 @@ "gridPos": { "h": 9, "w": 8, - "x": 8, - "y": 49 + "x": 0, + "y": 46 }, - "id": 41, - "interval": "$minstep", + "id": 10, "options": { "legend": { "calcs": [ @@ -2532,11 +2500,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2544,18 +2513,17 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "count(count(otelcol_process_cpu_seconds_total{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id))", + "exemplar": false, + "expr": "max(otelcol_exporter_queue_size{exporter=~\"$exporter\",job=\"$job\"}) by (exporter $grouping)", "format": "time_series", - "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Service instance count", + "legendFormat": "Max queue size: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Service Instance Count", + "title": "Exporter Queue Size", "type": "timeseries" }, { @@ -2563,7 +2531,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "", + "description": "Fixed capacity of the retry queue (in batches)", "fieldConfig": { "defaults": { "color": { @@ -2604,12 +2572,12 @@ }, "links": [], "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2617,18 +2585,17 @@ } ] }, - "unit": "s" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 9, "w": 8, - "x": 16, - "y": 49 + "x": 8, + "y": 46 }, - "id": 54, - "interval": "$minstep", + "id": 55, "options": { "legend": { "calcs": [ @@ -2641,11 +2608,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2653,18 +2621,17 @@ "uid": "$datasource" }, "editorMode": "code", - "exemplar": true, - "expr": "max(otelcol_process_uptime_total{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id)", + "exemplar": false, + "expr": "min(otelcol_exporter_queue_capacity{exporter=~\"$exporter\",job=\"$job\"}) by (exporter $grouping)", "format": "time_series", - "hide": false, "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "Service instance uptime: {{service_instance_id}}", + "legendFormat": "Queue capacity: {{exporter}} {{service_instance_id}}", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Uptime by Service Instance", + "title": "Exporter Queue Capacity", "type": "timeseries" }, { @@ -2676,23 +2643,50 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "inspect": false + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "links": [], "mappings": [], - "min": 0, + "max": 1, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2700,31 +2694,35 @@ } ] }, - "unit": "s" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 58 + "h": 9, + "w": 8, + "x": 16, + "y": 46 }, - "id": 57, - "interval": "$minstep", + "id": 67, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" + "legend": { + "calcs": [ + "min", + "max", + "mean" ], - "show": false + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2733,32 +2731,17 @@ }, "editorMode": "code", "exemplar": false, - "expr": "max(otelcol_process_uptime_total{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id,service_name,service_version)", - "format": "table", - "hide": false, - "instant": true, + "expr": "max(\r\n otelcol_exporter_queue_size{\r\n exporter=~\"$exporter\", job=\"$job\"\r\n }\r\n) by (exporter $grouping)\r\n/\r\nmin(\r\n otelcol_exporter_queue_capacity{\r\n exporter=~\"$exporter\", job=\"$job\"\r\n }\r\n) by (exporter $grouping)", + "format": "time_series", "interval": "$minstep", "intervalFactor": 1, - "legendFormat": "__auto", - "range": false, - "refId": "B" - } - ], - "title": "Service Instance Details", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - }, - "indexByName": {}, - "renameByName": {} - } + "legendFormat": "Queue capacity usage: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" } ], - "type": "table" + "title": "Exporter Queue Usage", + "type": "timeseries" }, { "collapsed": false, @@ -2766,11 +2749,11 @@ "h": 1, "w": 24, "x": 0, - "y": 63 + "y": 55 }, - "id": 59, + "id": 21, "panels": [], - "title": "Data Flows", + "title": "Collector", "type": "row" }, { @@ -2778,107 +2761,161 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "description": "Total physical memory (resident set size)", "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 64 - }, - "id": 58, - "options": { - "edges": {}, - "nodes": { - "mainStatUnit": "flops" - } - }, - "pluginVersion": "11.3.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": false, - "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_spans_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime_total{job=\"$job\"}[$__interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_spans_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "defaults": { + "color": { + "mode": "palette-classic" }, - "editorMode": "code", - "exemplar": false, - "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_spans_total{job=\"$job\"}[$__interval])) by (receiver))\r\n ,\"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n ,\"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_spans_total{job=\"$job\"}[$__interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "Spans Flow", - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "Value", - "renamePattern": "mainstat" - } - }, - { - "disabled": true, - "id": "calculateField", - "options": { - "alias": "secondarystat", - "mode": "reduceRow", - "reduce": { - "include": [ - "mainstat" - ], - "reducer": "sum" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] } - } - ], - "type": "nodeGraph" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", - "fieldConfig": { - "defaults": {}, - "overrides": [] + ] }, "gridPos": { "h": 9, "w": 8, - "x": 8, - "y": 64 + "x": 0, + "y": 56 }, - "id": 60, + "id": 40, + "interval": "$minstep", "options": { - "edges": {}, - "nodes": { - "mainStatUnit": "none" + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.3.0", + "pluginVersion": "11.5.2", "targets": [ { "datasource": { @@ -2887,170 +2924,3097 @@ }, "editorMode": "code", "exemplar": false, - "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_receiver_accepted_metric_points_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])\n ) by (receiver))\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime_total{job=\"$job\"}[$__interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_exporter_sent_metric_points_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])\n ) by (exporter))\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", - "format": "table", + "expr": "max(otelcol_process_memory_rss${suffix_bytes}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "editorMode": "code", - "exemplar": false, - "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_metric_points_total{job=\"$job\"}[$__interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_metric_points_total{job=\"$job\"}[$__interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "Metric Points Flow", - "transformations": [ - { - "id": "renameByRegex", + "editorMode": "code", + "exemplar": false, + "expr": "avg(otelcol_process_memory_rss${suffix_bytes}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(otelcol_process_memory_rss${suffix_bytes}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total RSS Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 52, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total Runtime Sys Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 56 + }, + "id": 53, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total Runtime Heap Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total CPU user and system time in percentage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg CPU usage " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min CPU usage " + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 39, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(rate(otelcol_process_cpu_seconds${suffix_total}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max CPU usage {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(rate(otelcol_process_cpu_seconds${suffix_total}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg CPU usage {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "min(rate(otelcol_process_cpu_seconds${suffix_total}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min CPU usage {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of service instances, which are reporting metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 41, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(count(otelcol_process_cpu_seconds${suffix_total}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id))", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Service instance count", + "range": true, + "refId": "B" + } + ], + "title": "Service Instance Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 54, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(otelcol_process_uptime${suffix_seconds}${suffix_total}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Service instance uptime: {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Uptime by Service Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 57, + "interval": "$minstep", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(otelcol_process_uptime${suffix_seconds}${suffix_total}{service_instance_id=~\".*\",job=\"$job\",service_version=\".+\"}) by (service_instance_id,service_name,service_version)\r\nor\r\nmax(\r\n otelcol_process_uptime${suffix_seconds}${suffix_total}{service_instance_id=~\".*\",job=\"$job\"} \r\n * on(job, instance) \r\n group_left(service_version) \r\n (\r\n target_info \r\n * on(job, instance) \r\n group_left \r\n label_replace(target_info{}, \"service_instance_id\", \"$1\", \"instance\", \"(.*)\")\r\n )\r\n) by (service_instance_id, service_name, service_version)", + "format": "table", + "hide": false, + "instant": true, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Service Instance Details", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 59, + "panels": [], + "title": "Signal flows", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 58, + "options": { + "edges": {}, + "nodes": { + "mainStatUnit": "flops" + }, + "zoomMode": "cooperative" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_spans${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix_seconds}${suffix_total}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_spans${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_spans${suffix_total}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n ,\"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n ,\"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_spans${suffix_total}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Spans Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "mainstat" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 80 + }, + "id": 60, + "options": { + "edges": {}, + "nodes": { + "mainStatUnit": "none" + }, + "zoomMode": "cooperative" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_receiver_accepted_metric_points${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (receiver))\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix_seconds}${suffix_total}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_exporter_sent_metric_points${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (exporter))\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_metric_points${suffix_total}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_metric_points${suffix_total}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Metric Points Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "Value #nodes" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 80 + }, + "id": 61, + "options": { + "edges": {}, + "nodes": { + "mainStatUnit": "flops" + }, + "zoomMode": "cooperative" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_log_records${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix_seconds}${suffix_total}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_log_records${suffix_total}{job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_log_records${suffix_total}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-edg-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_log_records${suffix_total}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n ,\"target\",\"-exp-\",\"transport\",\"exporter\"\r\n )\r\n ,\"source\",\"processor\",\"\",\"\"\r\n )\r\n ,\"id\",\"-edg-\",\"source\",\"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Log Records Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "mainstat" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + }, + "id": 79, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 88 + }, + "id": 32, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_filter_spans_filtered${suffix_total}{job=\"$job\"}[$__rate_interval])) by (filter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Filtered: {{filter}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Spans ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 88 + }, + "id": 81, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_filter_datapoints_filtered${suffix_total}{job=\"$job\"}[$__rate_interval])) by (filter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Filtered: {{filter}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Metric Points ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 88 + }, + "id": 82, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(${metric:value}(otelcol_processor_filter_log_records_filtered${suffix_total}{job=\"$job\"}[$__rate_interval])) by (filter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Filtered: {{filter}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Log Records ${metric:text}", + "type": "timeseries" + } + ], + "title": "Filter processors", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 90 + }, + "id": 68, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs. GRPC status codes: https://grpc.github.io/grpc/core/md_doc_statuscodes.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "displayName", + "value": "0 - OK" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1" + }, + "properties": [ + { + "id": "displayName", + "value": "1 - CANCELLED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2" + }, + "properties": [ + { + "id": "displayName", + "value": "2 - UNKNOWN" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3" + }, + "properties": [ + { + "id": "displayName", + "value": "3 - INVALID_ARGUMENT" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "displayName", + "value": "4 - DEADLINE_EXCEEDED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "displayName", + "value": "5 - NOT_FOUND" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "displayName", + "value": "6 - ALREADY_EXISTS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "7" + }, + "properties": [ + { + "id": "displayName", + "value": "7 - PERMISSION_DENIED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "displayName", + "value": "8 - RESOURCE_EXHAUSTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "9" + }, + "properties": [ + { + "id": "displayName", + "value": "9 - FAILED_PRECONDITION" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "displayName", + "value": "10 - ABORTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "11" + }, + "properties": [ + { + "id": "displayName", + "value": "11 - OUT_OF_RANGE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "displayName", + "value": "12 - UNIMPLEMENTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "13" + }, + "properties": [ + { + "id": "displayName", + "value": "13 - INTERNAL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "14" + }, + "properties": [ + { + "id": "displayName", + "value": "14 - UNAVAILABLE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "15" + }, + "properties": [ + { + "id": "displayName", + "value": "15 - DATA_LOSS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "16" + }, + "properties": [ + { + "id": "displayName", + "value": "16 - UNAUTHENTICATED" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 291 + }, + "id": 69, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(rpc_grpc_status_code) (${metric:value}(${prefix:raw}rpc_server_responses_per_rpc_count{job=\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC server responses by GRPC status code (receivers)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs. GRPC status codes: https://grpc.github.io/grpc/core/md_doc_statuscodes.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "displayName", + "value": "0 - OK" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1" + }, + "properties": [ + { + "id": "displayName", + "value": "1 - CANCELLED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2" + }, + "properties": [ + { + "id": "displayName", + "value": "2 - UNKNOWN" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3" + }, + "properties": [ + { + "id": "displayName", + "value": "3 - INVALID_ARGUMENT" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "displayName", + "value": "4 - DEADLINE_EXCEEDED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "displayName", + "value": "5 - NOT_FOUND" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "displayName", + "value": "6 - ALREADY_EXISTS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "7" + }, + "properties": [ + { + "id": "displayName", + "value": "7 - PERMISSION_DENIED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "displayName", + "value": "8 - RESOURCE_EXHAUSTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "9" + }, + "properties": [ + { + "id": "displayName", + "value": "9 - FAILED_PRECONDITION" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "displayName", + "value": "10 - ABORTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "11" + }, + "properties": [ + { + "id": "displayName", + "value": "11 - OUT_OF_RANGE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "displayName", + "value": "12 - UNIMPLEMENTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "13" + }, + "properties": [ + { + "id": "displayName", + "value": "13 - INTERNAL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "14" + }, + "properties": [ + { + "id": "displayName", + "value": "14 - UNAVAILABLE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "15" + }, + "properties": [ + { + "id": "displayName", + "value": "15 - DATA_LOSS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "16" + }, + "properties": [ + { + "id": "displayName", + "value": "16 - UNAUTHENTICATED" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 291 + }, + "id": 70, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(rpc_grpc_status_code) (${metric:value}(${prefix:raw}rpc_client_responses_per_rpc_count{job=\"$job\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC client responses by GRPC status code (exporters)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 300 + }, + "id": 72, + "interval": "$minstep", + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}rpc_server_duration_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC server duration (receivers)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 300 + }, + "id": 74, + "interval": "$minstep", + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}rpc_client_duration_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC client duration (exporters)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 308 + }, + "id": 73, + "interval": "$minstep", + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 0.1 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}rpc_server_request_size_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC server request size (receivers)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 308 + }, + "id": 75, + "interval": "$minstep", + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 0.1 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}rpc_client_request_size_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC client request size (exporters)", + "type": "heatmap" + } + ], + "title": "RPC server/client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 77, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 350 + }, + "id": 76, + "interval": "$minstep", + "maxDataPoints": 50, "options": { - "regex": "Value", - "renamePattern": "mainstat" - } + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}http_server_duration_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "__auto", + "refId": "B" + } + ], + "title": "HTTP server duration (receivers)", + "type": "heatmap" }, { - "disabled": true, - "id": "calculateField", + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 350 + }, + "id": 78, + "interval": "$minstep", + "maxDataPoints": 50, "options": { - "alias": "secondarystat", - "mode": "reduceRow", - "reduce": { - "include": [ - "Value #nodes" - ], - "reducer": "sum" + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" } - } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(${prefix:raw}http_client_duration_bucket{job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "HTTP client duration (exporters)", + "type": "heatmap" } ], - "type": "nodeGraph" + "title": "HTTP server/client", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 64 - }, - "id": 61, - "options": { - "edges": {}, - "nodes": { - "mainStatUnit": "flops" - } + "h": 1, + "w": 24, + "x": 0, + "y": 92 }, - "pluginVersion": "11.3.0", - "targets": [ + "id": 63, + "panels": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "editorMode": "code", - "exemplar": false, - "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_log_records_total{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime_total{job=\"$job\"}[$__interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_log_records_total{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "nodes" + "description": "Added: Number of namespace add events received.\nUpdated: Number of namespace update events received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*updated.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 212 + }, + "id": 64, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(otelcol_otelsvc_k8s_namespace_added${suffix_total}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Added: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(otelcol_otelsvc_k8s_namespace_updated${suffix_total}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Updated: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Namespaces", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "editorMode": "code", - "exemplar": false, - "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_log_records_total{job=\"$job\"}[$__interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-edg-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_log_records_total{job=\"$job\"}[$__interval])) by (exporter))\r\n ,\"target\",\"-exp-\",\"transport\",\"exporter\"\r\n )\r\n ,\"source\",\"processor\",\"\",\"\"\r\n )\r\n ,\"id\",\"-edg-\",\"source\",\"target\"\r\n)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "edges" - } - ], - "title": "Log Records Flow", - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "Value", - "renamePattern": "mainstat" - } - }, - { - "disabled": true, - "id": "calculateField", + "description": "Added: Number of pod add events received.\nUpdated: Number of pod update events received.\nDeleted: Number of pod delete events received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 212 + }, + "id": 65, + "interval": "$minstep", "options": { - "alias": "secondarystat", - "mode": "reduceRow", - "reduce": { - "include": [ - "mainstat" + "legend": { + "calcs": [ + "min", + "max", + "mean" ], - "reducer": "sum" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } - } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(otelcol_otelsvc_k8s_pod_added${suffix_total}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Added: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(otelcol_otelsvc_k8s_pod_updated${suffix_total}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Updated: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(otelcol_otelsvc_k8s_pod_deleted${suffix_total}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Deleted: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Pods", + "type": "timeseries" } ], - "type": "nodeGraph" + "title": "Kubernetes", + "type": "row" }, { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, + "collapsed": false, "gridPos": { - "h": 3, + "h": 1, "w": 24, "x": 0, - "y": 73 - }, - "id": 45, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": " | \nOTEL collector troubleshooting (how to enable telemetry metrics) | \nScaling the Collector (metrics to watch) | \nInstalled from Grafana.com dashboards", - "mode": "html" + "y": 93 }, - "pluginVersion": "11.3.0", + "id": 66, + "panels": [], "title": "Documentation", - "type": "text" + "type": "row" } ], "preload": false, - "refresh": "10s", + "refresh": "", "schemaVersion": 40, "tags": [ - "opentelemetry", - "monitoring" + "opentelemetry" ], "templating": { "list": [ @@ -3071,17 +6035,18 @@ "type": "prometheus", "uid": "$datasource" }, - "definition": "label_values(otelcol_process_uptime_total,job)", + "definition": "query_result({__name__=~\"otelcol_process_uptime.*\"})", "includeAll": false, "label": "Job", "name": "job", "options": [], "query": { - "query": "label_values(otelcol_process_uptime_total,job)", - "refId": "StandardVariableQuery" + "qryType": 3, + "query": "query_result({__name__=~\"otelcol_process_uptime.*\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": "", + "regex": "/.*{.*job=\"([a-zA-Z0-9_-]+)\".*}/", "sort": 1, "type": "query" }, @@ -3090,17 +6055,12 @@ "auto_count": 300, "auto_min": "10s", "current": { - "text": "auto", - "value": "$__auto_interval_minstep" + "text": "$__auto", + "value": "$__auto" }, "label": "Min step", "name": "minstep", "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_minstep" - }, { "selected": false, "text": "10s", @@ -3128,7 +6088,7 @@ }, { "current": { - "text": "Rate", + "text": "rate", "value": "rate" }, "includeAll": false, @@ -3156,37 +6116,40 @@ "type": "prometheus", "uid": "$datasource" }, - "definition": "label_values(receiver)", + "definition": "query_result(avg by (receiver) ({__name__=~\"otelcol_receiver_.+\",job=\"$job\"}))", "includeAll": true, "label": "Receiver", "name": "receiver", "options": [], "query": { - "query": "label_values(receiver)", - "refId": "StandardVariableQuery" + "qryType": 3, + "query": "query_result(avg by (receiver) ({__name__=~\"otelcol_receiver_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, - "regex": "", + "regex": "/.*receiver=\"(.*)\".*/", "sort": 1, "type": "query" }, { + "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "$datasource" }, - "definition": "label_values(processor)", + "definition": "query_result(avg by (processor) ({__name__=~\"otelcol_processor_.+\",job=\"$job\"}))", "includeAll": true, "label": "Processor", "name": "processor", "options": [], "query": { - "query": "label_values(processor)", - "refId": "StandardVariableQuery" + "qryType": 3, + "query": "query_result(avg by (processor) ({__name__=~\"otelcol_processor_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, - "regex": "", + "regex": "/.*processor=\"(.*)\".*/", "sort": 1, "type": "query" }, @@ -3197,23 +6160,24 @@ "type": "prometheus", "uid": "$datasource" }, - "definition": "label_values(exporter)", + "definition": "query_result(avg by (exporter) ({__name__=~\"otelcol_exporter_.+\",job=\"$job\"}))", "includeAll": true, "label": "Exporter", "name": "exporter", "options": [], "query": { - "query": "label_values(exporter)", - "refId": "StandardVariableQuery" + "qryType": 3, + "query": "query_result(avg by (exporter) ({__name__=~\"otelcol_exporter_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, - "regex": "", + "regex": "/.*exporter=\"(.*)\".*/", "sort": 1, "type": "query" }, { "current": { - "text": "None (basic metrics)", + "text": "", "value": "" }, "description": "Detailed metrics must be configured in the collector configuration. They add grouping by transport protocol (http/grpc) for receivers. ", @@ -3239,17 +6203,116 @@ ], "query": "None (basic metrics) : , By transport (detailed metrics) : \\,transport, By service instance id : \\,service_instance_id", "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "description": "Some exporter(s) configuration(s) may add the metric suffix _total. This variable will detect this case.", + "hide": 2, + "includeAll": false, + "label": "Suffix _total", + "name": "suffix_total", + "options": [], + "query": { + "qryType": 3, + "query": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/.*(_total)+{.*/", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "description": "Some exporter(s) configuration(s) may add the metric suffix _seconds_total. This variable will detect this \"_seconds\" suffix part.", + "hide": 2, + "includeAll": false, + "label": "Suffix _seconds", + "name": "suffix_seconds", + "options": [], + "query": { + "qryType": 3, + "query": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/otelcol_process_uptime(.*)_total{.*/", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result({__name__=~\"otelcol_process_memory_rss.+\",job=\"$job\"})", + "description": "Some exporter(s) configuration(s) may add the metric suffix _bytes. This variable will detect this \"_bytes\" suffix part.", + "hide": 2, + "includeAll": false, + "label": "Suffix _bytes", + "name": "suffix_bytes", + "options": [], + "query": { + "qryType": 3, + "query": "query_result({__name__=~\"otelcol_process_memory_rss.+\",job=\"$job\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/otelcol_process_memory_rss(.*){.*/", + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result({__name__=~\".*.*rpc_server_duration_bucket|.*rpc_client_duration_bucket|.*http_server_duration_bucket|.*http_client_duration_bucket.*\",job=\"$job\"})", + "description": "Some metrics (e.g., RPC, HTTP) may no longer have the otelcol_ prefix. This will detect it. See https://github.com/open-telemetry/opentelemetry-collector/pull/9759", + "hide": 2, + "includeAll": false, + "label": "Prefix", + "name": "prefix", + "options": [], + "query": { + "qryType": 3, + "query": "query_result({__name__=~\".*.*rpc_server_duration_bucket|.*rpc_client_duration_bucket|.*http_server_duration_bucket|.*http_client_duration_bucket.*\",job=\"$job\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/(.*)(rpc|http)_(server|client)_duration_bucket.*{.*/", + "type": "query" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "filters": [], + "label": "Ad Hoc", + "name": "adhoc", + "type": "adhoc" } ] }, "time": { - "from": "now-6h", + "from": "now-15m", "to": "now" }, "timepicker": {}, - "timezone": "utc", + "timezone": "", "title": "OpenTelemetry Collector", "uid": "BKf2sowmj", - "version": 2, + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/src/grafana/provisioning/dashboards/demo/spanmetrics-dashboard.json b/src/grafana/provisioning/dashboards/demo/spanmetrics-dashboard.json index c7a11f1c10..4a26602786 100644 --- a/src/grafana/provisioning/dashboards/demo/spanmetrics-dashboard.json +++ b/src/grafana/provisioning/dashboards/demo/spanmetrics-dashboard.json @@ -1022,7 +1022,7 @@ ] }, "time": { - "from": "now-1h", + "from": "now-15m", "to": "now" }, "timepicker": {}, diff --git a/src/grafana/provisioning/datasources/opensearch.yaml b/src/grafana/provisioning/datasources/opensearch.yaml index df633bc743..814250ae75 100644 --- a/src/grafana/provisioning/datasources/opensearch.yaml +++ b/src/grafana/provisioning/datasources/opensearch.yaml @@ -5,6 +5,7 @@ apiVersion: 1 datasources: - name: OpenSearch + uid: webstore-logs type: grafana-opensearch-datasource url: http://opensearch:9200/ access: proxy @@ -13,7 +14,7 @@ datasources: jsonData: database: otel flavor: opensearch - logLevelField: severity + logLevelField: severity.text.keyword logMessageField: body pplEnabled: true timeField: observedTimestamp diff --git a/src/otel-collector/otelcol-config.yml b/src/otel-collector/otelcol-config.yml index 248c372c4f..2449846e20 100644 --- a/src/otel-collector/otelcol-config.yml +++ b/src/otel-collector/otelcol-config.yml @@ -79,14 +79,6 @@ receivers: mute_process_exe_error: true mute_process_io_error: true mute_process_user_error: true - # Collector metrics - prometheus: - config: - scrape_configs: - - job_name: 'otel-collector' - scrape_interval: 10s - static_configs: - - targets: ['0.0.0.0:8888'] exporters: debug: @@ -107,6 +99,10 @@ exporters: processors: batch: + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 25 transform: error_mode: ignore trace_statements: @@ -123,13 +119,24 @@ service: pipelines: traces: receivers: [otlp] - processors: [transform, batch] + processors: [memory_limiter, transform, batch] exporters: [otlp, debug, spanmetrics] metrics: - receivers: [hostmetrics, docker_stats, httpcheck/frontend-proxy, otlp, prometheus, redis, spanmetrics] - processors: [batch] + receivers: [hostmetrics, docker_stats, httpcheck/frontend-proxy, otlp, redis, spanmetrics] + processors: [memory_limiter, batch] exporters: [otlphttp/prometheus, debug] logs: receivers: [otlp] - processors: [batch] + processors: [memory_limiter, batch] exporters: [opensearch, debug] + telemetry: + metrics: + level: detailed + readers: + - periodic: + interval: 10000 + timeout: 5000 + exporter: + otlp: + protocol: grpc + endpoint: ${env:OTEL_COLLECTOR_HOST}:${env:OTEL_COLLECTOR_PORT_GRPC} diff --git a/src/prometheus/prometheus-config.yaml b/src/prometheus/prometheus-config.yaml index 736212af42..7ba23eedea 100644 --- a/src/prometheus/prometheus-config.yaml +++ b/src/prometheus/prometheus-config.yaml @@ -16,12 +16,6 @@ otlp: - container.name - deployment.environment.name -scrape_configs: - - job_name: otel-collector - static_configs: - - targets: - - 'otel-collector:8888' - storage: tsdb: out_of_order_time_window: 30m