Make the daemonset rollout stuck alert configurable. (#989)

* Make the daemonset rollout stuck alert configurable. For bigger Kubernetes clusters with bigger node churn (for instance, cloud clusters with spot nodes), the daemonset rollouts often get stuck for longer than just 15 minutes. Since the alert might easily misfire even in cases where the delay is legitimate. This PR introduces configurable `for` value to allow for customization. As a default, the original value `15m` is left, so the only real difference would be a slight change in the alert message formatting. Signed-off-by: Milan Plzik <[email protected]> * Fix the tests. Signed-off-by: Milan Plzik <[email protected]> --------- Signed-off-by: Milan Plzik <[email protected]>
kubernetes-monitoring · Nov 27, 2024 · a3fbf21 · a3fbf21
1 parent 72a1a23
commit a3fbf21
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 6 deletions.
diff --git a/alerts/apps_alerts.libsonnet b/alerts/apps_alerts.libsonnet
@@ -4,6 +4,7 @@ local utils = import '../lib/utils.libsonnet';
   _config+:: {
     kubeStateMetricsSelector: error 'must provide selector for kube-state-metrics',
     kubeJobTimeoutDuration: error 'must provide value for kubeJobTimeoutDuration',
+    kubeDaemonSetRolloutStuckFor: '15m',
     namespaceSelector: null,
     prefixedNamespaceSelector: if self.namespaceSelector != null then self.namespaceSelector + ',' else '',
   },
@@ -204,10 +205,10 @@ local utils = import '../lib/utils.libsonnet';
               severity: 'warning',
             },
             annotations: {
-              description: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.',
+              description: 'DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least %(kubeDaemonSetRolloutStuckFor)s.' % $._config,
               summary: 'DaemonSet rollout is stuck.',
             },
-            'for': '15m',
+            'for': $._config.kubeDaemonSetRolloutStuckFor,
           },
           {
             expr: |||

diff --git a/tests.yaml b/tests.yaml
@@ -822,7 +822,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "DaemonSet rollout is stuck."
-        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15 minutes.'
+        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15m.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
   - eval_time: 34m
     alertname: KubeDaemonSetRolloutStuck
@@ -878,7 +878,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "DaemonSet rollout is stuck."
-        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15 minutes.'
+        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15m.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
   - eval_time: 34m
     alertname: KubeDaemonSetRolloutStuck
@@ -909,7 +909,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "DaemonSet rollout is stuck."
-        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15 minutes.'
+        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15m.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
   - eval_time: 34m
     alertname: KubeDaemonSetRolloutStuck
@@ -940,7 +940,7 @@ tests:
         severity: warning
       exp_annotations:
         summary: "DaemonSet rollout is stuck."
-        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15 minutes.'
+        description: 'DaemonSet monitoring/node-exporter has not finished or progressed for at least 15m.'
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
   - eval_time: 36m
     alertname: KubeDaemonSetRolloutStuck