improve monitoring

fastmachinelearning · Feb 13, 2025 · 1e12681 · 1e12681
1 parent 0645653
commit 1e12681
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 17 deletions.
diff --git a/helm/supersonic/dashboards/default.json b/helm/supersonic/dashboards/default.json
@@ -112,7 +112,7 @@
             },
             "disableTextWrap": false,
             "editorMode": "code",
-            "expr": "sum by(release)(envoy_cluster_membership_total{envoy_cluster_name=\"triton_grpc_service\", release=~\"${release_name}\"})",
+            "expr": "sum by(release)(envoy_cluster_membership_total{envoy_cluster_name=\"triton_grpc_service\", release=~\"${release_name}\", namespace=~\"${namespace}\"})",
             "fullMetaSearch": false,
             "includeNullMetadata": true,
             "instant": false,
@@ -328,7 +328,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "sum by (release) (    rate(nv_inference_compute_infer_duration_us{release=~\"${release_name}\"}[15s]))  /sum by (release) (    (rate(nv_inference_exec_count{release=~\"${release_name}\"}[15s]) * 1000) + 0.001)",
+            "expr": "sum (rate(nv_inference_compute_infer_duration_us{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) / sum ((rate(nv_inference_exec_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s]) * 1000))",
             "instant": false,
             "legendFormat": "Inference",
             "range": true,
@@ -340,7 +340,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "sum by (release) (    rate(nv_inference_queue_duration_us{release=~\"${release_name}\"}[15s]))  /sum by (release) (    (rate(nv_inference_exec_count{release=~\"${release_name}\"}[15s]) * 1000) + 0.001)",
+            "expr": "sum (rate(nv_inference_queue_duration_us{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) / sum ((rate(nv_inference_exec_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s]) * 1000))",
             "hide": false,
             "instant": false,
             "legendFormat": "Queue",
@@ -353,7 +353,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "sum by (release) (    rate(nv_inference_compute_input_duration_us{release=~\"${release_name}\"}[15s]))  /sum by (release) (    (rate(nv_inference_exec_count{release=~\"${release_name}\"}[15s]) * 1000) + 0.001)",
+            "expr": "sum (rate(nv_inference_compute_input_duration_us{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) / sum ((rate(nv_inference_exec_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s]) * 1000))",
             "hide": false,
             "instant": false,
             "legendFormat": "Input",
@@ -366,7 +366,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "sum by (release) (    rate(nv_inference_compute_output_duration_us{release=~\"${release_name}\"}[15s]))  /sum by (release) (    (rate(nv_inference_exec_count{release=~\"${release_name}\"}[15s]) * 1000) + 0.001)",
+            "expr": "sum (rate(nv_inference_compute_output_duration_us{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) / sum ((rate(nv_inference_exec_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s]) * 1000))",
             "hide": false,
             "instant": false,
             "legendFormat": "Output",
@@ -379,7 +379,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "  sum(\n    rate(envoy_http_downstream_rq_time_sum{envoy_http_conn_manager_prefix=\"ingress_grpc\", release=~\"${release_name}\"}[15s])\n    /\n    rate(envoy_http_downstream_rq_time_count{envoy_http_conn_manager_prefix=\"ingress_grpc\", release=~\"${release_name}\"}[15s])\n  ) by (release)",
+            "expr": "sum(rate(envoy_http_downstream_rq_time_sum{envoy_http_conn_manager_prefix=\"ingress_grpc\", release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) / sum(rate(envoy_http_downstream_rq_time_count{envoy_http_conn_manager_prefix=\"ingress_grpc\", release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s]))",
             "hide": false,
             "instant": false,
             "legendFormat": "Total (measured at proxy)",
@@ -477,7 +477,7 @@
             },
             "disableTextWrap": false,
             "editorMode": "code",
-            "expr": "sum by(release)(envoy_cluster_membership_total{envoy_cluster_name=\"triton_grpc_service\", release=~\"${release_name}\"})",
+            "expr": "sum by(release)(envoy_cluster_membership_total{envoy_cluster_name=\"triton_grpc_service\", release=~\"${release_name}\", namespace=~\"${namespace}\"})",
             "fullMetaSearch": false,
             "includeNullMetadata": true,
             "instant": false,
@@ -600,7 +600,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "nv_gpu_utilization",
+            "expr": "nv_gpu_utilization{namespace=~\"${namespace}\"}",
             "hide": false,
             "instant": false,
             "legendFormat": "{{pod}}",
@@ -708,7 +708,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "nv_gpu_power_usage / nv_gpu_power_limit",
+            "expr": "(nv_gpu_power_usage{namespace=~\"${namespace}\"} / nv_gpu_power_limit{namespace=~\"${namespace}\"})",
             "hide": false,
             "instant": false,
             "legendFormat": "{{pod}}",
@@ -762,6 +762,7 @@
               }
             },
             "mappings": [],
+            "max": 1,
             "min": 0,
             "thresholds": {
               "mode": "absolute",
@@ -784,7 +785,7 @@
                 }
               ]
             },
-            "unit": "bytes"
+            "unit": "percentunit"
           },
           "overrides": []
         },
@@ -815,7 +816,7 @@
               "uid": "prometheus"
             },
             "editorMode": "code",
-            "expr": "nv_gpu_memory_used_bytes",
+            "expr": "nv_gpu_memory_used_bytes{namespace=~\"${namespace}\"}",
             "hide": false,
             "instant": false,
             "legendFormat": "{{pod}}",
@@ -925,7 +926,7 @@
             },
             "editorMode": "code",
             "exemplar": false,
-            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\"}[15s]))by(release)",
+            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) by(release)",
             "instant": false,
             "interval": "",
             "legendFormat": "{{ release }}",
@@ -1022,7 +1023,7 @@
             },
             "editorMode": "code",
             "exemplar": false,
-            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\"}[15s]))by(pod)",
+            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) by(pod)",
             "instant": false,
             "interval": "",
             "legendFormat": "{{ pod }}",
@@ -1119,7 +1120,7 @@
             },
             "editorMode": "code",
             "exemplar": false,
-            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\"}[15s]))by(model)",
+            "expr": "sum (rate(nv_inference_count{release=~\"${release_name}\", namespace=~\"${namespace}\"}[15s])) by(model)",
             "instant": false,
             "interval": "",
             "legendFormat": "{{ model }}",

diff --git a/helm/supersonic/dashboards/variables.json b/helm/supersonic/dashboards/variables.json
@@ -14,7 +14,7 @@
           "type": "prometheus",
           "uid": "prometheus"
         },
-        "definition": "label_values(release)",
+        "definition": "label_values({app=\"supersonic\",namespace=\"%NAMESPACE%\"},release)",
         "hide": 0,
         "includeAll": true,
         "label": "Release",
@@ -23,7 +23,7 @@
         "options": [],
         "query": {
           "qryType": 1,
-          "query": "label_values(release)",
+          "query": "label_values({app=\"supersonic\",namespace=\"%NAMESPACE%\"},release)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -63,6 +63,14 @@
         "query": "%PROMETHEUS_URL_FULL%",
         "skipUrlSync": false,
         "type": "constant"
+      },
+      {
+        "hide": 2,
+        "label": "Namespace",
+        "name": "namespace",
+        "query": "%NAMESPACE%",
+        "skipUrlSync": false,
+        "type": "constant"
       }
     ]
   }

diff --git a/helm/supersonic/templates/default-dashboard.yaml b/helm/supersonic/templates/default-dashboard.yaml
@@ -25,6 +25,7 @@ data:
     {{- /* Replace template variables with actual values */ -}}
     {{- $dashboard := $dashboard | replace "%RELEASE_NAME%" .Release.Name }}
     {{- $dashboard := $dashboard | replace "%CHART_VERSION%" .Chart.Version }}
+    {{- $dashboard := $dashboard | replace "%NAMESPACE%" .Release.Namespace }}
     {{- $dashboard := $dashboard | replace "%SERVER_LOAD_METRIC%" $metric }}
     {{- $dashboard := $dashboard | replace "%SERVER_LOAD_THRESHOLD%" $threshold }}
     {{- $dashboard := $dashboard | replace "%PROMETHEUS_URL_FULL%" $prometheus_url }}

diff --git a/helm/supersonic/templates/servicemonitor.yaml b/helm/supersonic/templates/servicemonitor.yaml
@@ -22,6 +22,10 @@ spec:
         targetLabel: release
         regex: "(.*)"
         replacement: "{{ include "supersonic.name" . }}"
+      - sourceLabels: [__address__]
+        targetLabel: app
+        regex: "(.*)"
+        replacement: "{{ .Chart.Name }}"
 
 ---
 
@@ -48,4 +52,8 @@ spec:
       - sourceLabels: [__address__]
         targetLabel: release
         regex: "(.*)"
-        replacement: "{{ include "supersonic.name" . }}"
+        replacement: "{{ include "supersonic.name" . }}"
+      - sourceLabels: [__address__]
+        targetLabel: app
+        regex: "(.*)"
+        replacement: "{{ .Chart.Name }}"