diff --git a/hack/monitoring/grafana-values.yaml b/hack/monitoring/grafana-values.yaml index cc3dc405a..bfc334541 100644 --- a/hack/monitoring/grafana-values.yaml +++ b/hack/monitoring/grafana-values.yaml @@ -1,5 +1,9 @@ nodeSelector: kubernetes.azure.com/mode: system +tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule datasources: datasources.yaml: apiVersion: 1 @@ -24,7 +28,6 @@ dashboardProviders: dashboards: default: capacity-dashboard: - url: https://karpenter.sh/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + url: https://karpenter.sh/v1.0/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json performance-dashboard: - # url: https://karpenter.sh/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json - url: https://raw.githubusercontent.com/Azure/karpenter-provider-azure/main/hack/monitoring/karpenter-performance-dashboard.json + url: https://karpenter.sh/v1.0/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/hack/monitoring/karpenter-performance-dashboard.json b/hack/monitoring/karpenter-performance-dashboard.json deleted file mode 100644 index 4f624f795..000000000 --- a/hack/monitoring/karpenter-performance-dashboard.json +++ /dev/null @@ -1,761 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": true, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_nodes_termination_time_seconds{quantile=\"0\"}", - "legendFormat": "Min", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_nodes_termination_time_seconds{quantile=\"0.5\"}", - "hide": false, - "legendFormat": "P50", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_nodes_termination_time_seconds{quantile=\"0.9\"}", - "hide": false, - "legendFormat": "P90", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_nodes_termination_time_seconds{quantile=\"0.99\"}", - "hide": false, - "legendFormat": "P99", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_nodes_termination_time_seconds{quantile=\"1\"}", - "hide": false, - "legendFormat": "Max", - "range": true, - "refId": "E" - } - ], - "title": "Node Termination Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_pods_startup_time_seconds{quantile=\"0\"}", - "format": "time_series", - "legendFormat": "Min", - "range": true, - "refId": "Minimum" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_pods_startup_time_seconds{quantile=\"0.5\"}", - "hide": false, - "legendFormat": "P50", - "range": true, - "refId": "Median" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_pods_startup_time_seconds{quantile=\"0.9\"}", - "hide": false, - "legendFormat": "P90", - "range": true, - "refId": "P90" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_pods_startup_time_seconds{quantile=\"0.99\"}", - "hide": false, - "legendFormat": "P99", - "range": true, - "refId": "P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "karpenter_pods_startup_time_seconds{quantile=\"1\"}", - "hide": false, - "legendFormat": "Max", - "range": true, - "refId": "Maximum" - } - ], - "title": "Pod Startup Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 7, - "x": 0, - "y": 17 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0, rate(karpenter_cloudprovider_duration_seconds_bucket{method=\"$method\"}[10m])) ", - "hide": false, - "legendFormat": "Min", - "range": true, - "refId": "Minimum" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, rate(karpenter_cloudprovider_duration_seconds_bucket{method=\"$method\"}[10m])) ", - "legendFormat": "P50", - "range": true, - "refId": "Median" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, rate(karpenter_cloudprovider_duration_seconds_bucket{method=\"$method\"}[10m])) ", - "hide": false, - "legendFormat": "P90", - "range": true, - "refId": "P90" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(karpenter_cloudprovider_duration_seconds_bucket{method=\"$method\"}[10m])) ", - "hide": false, - "legendFormat": "P99", - "range": true, - "refId": "P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1, rate(karpenter_cloudprovider_duration_seconds_bucket{method=\"$method\"}[10m])) ", - "hide": false, - "legendFormat": "Max", - "range": true, - "refId": "Maximum" - } - ], - "title": "Provider Method Duration [$method]", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 7, - "y": 17 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", - "hide": false, - "legendFormat": "Min", - "range": true, - "refId": "Minimum" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", - "legendFormat": "P50", - "range": true, - "refId": "Median" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", - "hide": false, - "legendFormat": "P90", - "range": true, - "refId": "P90" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", - "hide": false, - "legendFormat": "P99", - "range": true, - "refId": "P99" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", - "hide": false, - "legendFormat": "Max", - "range": true, - "refId": "Maximum" - } - ], - "title": "Controller Reconciliation Latency [$controller]", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 17 - }, - "id": 8, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "sum(rate(controller_runtime_reconcile_total[10m])) by (controller)", - "legendFormat": "{{controller}}", - "range": true, - "refId": "A" - } - ], - "title": "Controller Reconciliation Rate", - "type": "bargauge" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "PBFA97CFB590B2093" - }, - "hide": 0, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "nodepool.hash", - "value": "nodepool.hash" - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "controller", - "options": [], - "query": { - "query": "label_values(controller_runtime_reconcile_time_seconds_count, controller)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": { - "selected": false, - "text": "Create", - "value": "Create" - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(karpenter_cloudprovider_duration_seconds_count,method)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(karpenter_cloudprovider_duration_seconds_count,method)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Karpenter & Provider Performance", - "uid": "_bdgC2g4zP", - "version": 2, - "weekStart": "" - } \ No newline at end of file diff --git a/hack/monitoring/prometheus-values.yaml b/hack/monitoring/prometheus-values.yaml index 0e618197c..75687a0c9 100644 --- a/hack/monitoring/prometheus-values.yaml +++ b/hack/monitoring/prometheus-values.yaml @@ -4,10 +4,20 @@ server: enabled: false nodeSelector: kubernetes.azure.com/mode: system + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule alertmanager: persistentVolume: enabled: false + nodeSelector: + kubernetes.azure.com/mode: system + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule kube-state-metrics: enabled: false @@ -22,10 +32,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - karpenter + - kube-system relabel_configs: - source_labels: - __meta_kubernetes_endpoints_name - __meta_kubernetes_endpoint_port_name action: keep - regex: karpenter;http-metrics \ No newline at end of file + regex: karpenter;http-metrics