test: added unit tests for kserve

red-hat-data-services · Sep 16, 2024 · 49a8afc · 49a8afc
1 parent b5f3b2d
commit 49a8afc
Showing 1 changed file with 83 additions and 0 deletions.
diff --git a/tests/prometheus_unit_tests/kserve_alerts_unit_tests.yaml b/tests/prometheus_unit_tests/kserve_alerts_unit_tests.yaml
@@ -0,0 +1,83 @@
+rule_files:
+  - "kserve_alerts.yaml"
+
+evaluation_interval: 1m
+
+tests:
+  # burn rate
+  - interval: 1m
+    input_series:
+      - series: probe_success:burnrate5m{instance="kserve-controller-manager"}
+        values: "0x60"
+      - series: probe_success:burnrate30m{instance="kserve-controller-manager"}
+        values: "0x60"
+      - series: probe_success:burnrate1h{instance="kserve-controller-manager"}
+        values: "0x60"
+      - series: probe_success:burnrate2hm{instance="kserve-controller-manager"}
+        values: "0x60"
+      - series: probe_success:burnrate6h{instance="kserve-controller-manager"}
+        values: "0x60"
+      - series: probe_success:burnrate1d{instance="kserve-controller-manager"}
+        values: "0x60"
+    alert_rule_test:
+      - eval_time: 1h
+        alertname: Kserve Controller Probe Success Burn Rate
+        exp_alerts: []
+
+  - interval: 1m
+    input_series:
+      - series: probe_success:burnrate5m{instance="kserve-controller-manager"}
+        values: "1+1x60"
+      - series: probe_success:burnrate1h{instance="kserve-controller-manager"}
+        values: "1+1x60"
+    alert_rule_test:
+      - eval_time: 2m
+        alertname: Kserve Controller Probe Success Burn Rate
+        exp_alerts:
+          - exp_labels:
+              alertname: Kserve Controller Probe Success Burn Rate
+              instance: "kserve-controller-manager"
+              severity: critical
+            exp_annotations:
+              message: "High error budget burn for kserve-controller-manager (current value: 3)."
+              summary: Kserve Controller Probe Success Burn Rate
+              triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
+
+  - interval: 1m
+    input_series:
+      - series: probe_success:burnrate30m{instance="kserve-controller-manager"}
+        values: "1+1x60"
+      - series: probe_success:burnrate6h{instance="kserve-controller-manager"}
+        values: "1+1x60"
+    alert_rule_test:
+      - eval_time: 15m
+        alertname: Kserve Controller Probe Success Burn Rate
+        exp_alerts:
+          - exp_labels:
+              alertname: Kserve Controller Probe Success Burn Rate
+              instance: "kserve-controller-manager"
+              severity: critical
+            exp_annotations:
+              message: "High error budget burn for kserve-controller-manager (current value: 16)."
+              summary: Kserve Controller Probe Success Burn Rate
+              triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
+
+  - interval: 1m
+    input_series:
+      - series: probe_success:burnrate2h{instance="kserve-controller-manager"}
+        values: "1+1x60"
+      - series: probe_success:burnrate1d{instance="kserve-controller-manager"}
+        values: "1+1x60"
+    alert_rule_test:
+      - eval_time: 1h
+        alertname: Kserve Controller Probe Success Burn Rate
+        exp_alerts:
+          - exp_labels:
+              alertname: Kserve Controller Probe Success Burn Rate
+              instance: "kserve-controller-manager"
+              severity: warning
+            exp_annotations:
+              message: "High error budget burn for kserve-controller-manager (current value: 61)."
+              summary: Kserve Controller Probe Success Burn Rate
+              triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
+