[main] Implement alerting

charts/rancher-backup/templates/prometheus-rules.yaml

@@ -0,0 +1,36 @@
+{{ if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" }}
+{{ if or .Values.monitoring.prometheusRules.customRules.enabled .Values.monitoring.prometheusRules.defaultAlert.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "backupRestore.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  groups:
+    {{- if .Values.monitoring.prometheusRules.defaultAlert.enabled }}
+    - name: backup-restore
+      rules:
+      - record: status:rancher_backups_attempted_total
+        expr: rancher_backups_attempted_total * on(name) group_left(status) rancher_backup_info
+      - record: status:rancher_backups_failed_total
+        expr: rancher_backups_failed_total * on(name) group_left(status) rancher_backup_info
+      - alert: BackupFailed
+        expr: (sum(rate(status:rancher_backups_attempted_total[{{ .Values.monitoring.prometheusRules.defaultAlert.window }}])) by (status) / (sum(rate(status:rancher_backups_attempted_total[{{ .Values.monitoring.prometheusRules.defaultAlert.window }}])) by (status) - sum(rate(status:rancher_backups_failed_total[{{ .Values.monitoring.prometheusRules.defaultAlert.window }}])) by (status))) > 1
+        for: 1m
+        {{- with .Values.monitoring.prometheusRules.defaultAlert.labels }}
+        labels:
+            {{- toYaml . | nindent 10 }}
+        {{- end }}
+        annotations:
+          summary: "Backup failed due to error: {{ "{{" }} $labels.status {{ "}}" }}"
+          description: "The rancher-backup operator has failed to process a backup. Error: {{ "{{" }} $labels.status {{ "}}" }}"
+    {{- end }}
+    {{- if .Values.monitoring.prometheusRules.customRules.enabled }}
+    - name: backup-restore-custom
+      {{- with .Values.monitoring.prometheusRules.customRules.rules }}
+      rules:
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    {{- end }}
+{{- end }}
+{{- end }}

charts/rancher-backup/values.yaml

@@ -90,3 +90,17 @@ monitoring:
     additionalLabels: {}
     metricRelabelings: []
     relabelings: []
+
+  prometheusRules:
+
+    ## The "BackupFailed" alert is included by default when .Values.monitoring.prometheusRules.defaultAlert.enabled is set to true and rancher-monitoring is installed.
+    defaultAlert:
+      enabled: false
+      labels:
+        severity: critical
+      window: 5m
+
+    ## Define custom Prometheus rules here.
+    customRules:
+      enabled: false
+      rules: []

e2e/backup/backup_test.go

@@ -83,27 +83,27 @@ func formatBackupMetrics(backups []string) string {
 	metrics += fmt.Sprintf("rancher_backup_count %d", len(backups))
 
 	rancherBackupsAttemptedHeader := fmt.Sprint(`
-	# HELP rancher_backups_attempted Number of Rancher Backups processed by this operator
-	# TYPE rancher_backups_attempted counter
+	# HELP rancher_backups_attempted_total Number of Rancher Backups processed by this operator
+	# TYPE rancher_backups_attempted_total counter
 	`)
 
 	metrics += rancherBackupsAttemptedHeader
 	for _, b := range backups {
 		if b == s3Recurring {
-			metrics += fmt.Sprintf("rancher_backups_attempted{name=\"%s\"} 2\n", b)
+			metrics += fmt.Sprintf("rancher_backups_attempted_total{name=\"%s\"} 2\n", b)
 		} else {
-			metrics += fmt.Sprintf("rancher_backups_attempted{name=\"%s\"} 1\n", b)
+			metrics += fmt.Sprintf("rancher_backups_attempted_total{name=\"%s\"} 1\n", b)
 		}
 	}
 
 	rancherBackupsFailedHeader := fmt.Sprint(`
-	# HELP rancher_backups_failed Number of failed Rancher Backups processed by this operator
-	# TYPE rancher_backups_failed counter
+	# HELP rancher_backups_failed_total Number of failed Rancher Backups processed by this operator
+	# TYPE rancher_backups_failed_total counter
 	`)
 
 	metrics += rancherBackupsFailedHeader
 	for _, b := range backups {
-		metrics += fmt.Sprintf("rancher_backups_failed{name=\"%s\"} 0\n", b)
+		metrics += fmt.Sprintf("rancher_backups_failed_total{name=\"%s\"} 0\n", b)
 	}
 
 	return metrics + "\n"
@@ -113,8 +113,8 @@ func formatBackupMetadataMetrics(backups []backupv1.Backup) string {
 	var metrics string
 
 	rancherBackupHeader := fmt.Sprint(`
-	# HELP rancher_backup Details on a specific Rancher Backup CR
-	# TYPE rancher_backup gauge
+	# HELP rancher_backup_info Details on a specific Rancher Backup CR
+	# TYPE rancher_backup_info gauge
 	`)
 
 	metrics += rancherBackupHeader
@@ -133,7 +133,7 @@ func formatBackupMetadataMetrics(backups []backupv1.Backup) string {
 		}
 
 		metrics += fmt.Sprintf(`
-		rancher_backup{backupType="%s",filename="%s",lastSnapshot="%s",name="%s",nextSnapshot="%s",resourceSetName="%s",retentionCount="%d",status="%s",storageLocation="%s"} 1
+		rancher_backup_info{backupType="%s",filename="%s",lastSnapshot="%s",name="%s",nextSnapshot="%s",resourceSetName="%s",retentionCount="%d",status="%s",storageLocation="%s"} 1
 		`, backupType, b.Status.Filename, b.Status.LastSnapshotTS, b.Name, backupNextSnapshot, b.Spec.ResourceSetName, b.Spec.RetentionCount, backupMessage, b.Status.StorageLocation)
 	}
 
@@ -248,8 +248,8 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
 					"rancher_backup_count",
-					"rancher_backups_attempted",
-					"rancher_backups_failed",
+					"rancher_backups_attempted_total",
+					"rancher_backups_failed_total",
 				)
 			}).Should(Succeed())
 		})
@@ -293,8 +293,8 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
 					"rancher_backup_count",
-					"rancher_backups_attempted",
-					"rancher_backups_failed",
+					"rancher_backups_attempted_total",
+					"rancher_backups_failed_total",
 				)
 			}).Should(Succeed())
 		})
@@ -379,8 +379,8 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
 					"rancher_backup_count",
-					"rancher_backups_attempted",
-					"rancher_backups_failed",
+					"rancher_backups_attempted_total",
+					"rancher_backups_failed_total",
 				)
 			}).Should(Succeed())
 		})
@@ -452,8 +452,8 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
 					"rancher_backup_count",
-					"rancher_backups_attempted",
-					"rancher_backups_failed",
+					"rancher_backups_attempted_total",
+					"rancher_backups_failed_total",
 				)
 			}).Should(Succeed())
 		})
@@ -528,8 +528,8 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
 					"rancher_backup_count",
-					"rancher_backups_attempted",
-					"rancher_backups_failed",
+					"rancher_backups_attempted_total",
+					"rancher_backups_failed_total",
 				)
 			}).Should(Succeed())
 		})
@@ -566,7 +566,7 @@ var _ = Describe("Backup e2e remote", Ordered, Label("integration"), func() {
 				expected := formatBackupMetadataMetrics(backups.Items)
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
-					"rancher_backup",
+					"rancher_backup_info",
 				)
 			}).Should(Succeed())
 		})

e2e/backup/restore_test.go

@@ -45,8 +45,8 @@ func formatRestoreMetadataMetrics(restores []backupv1.Restore) string {
 	var metrics string
 
 	rancherRestoreHeader := fmt.Sprint(`
-	# HELP rancher_restore Details on a specific Rancher Restore CR
-	# TYPE rancher_restore gauge
+	# HELP rancher_restore_info Details on a specific Rancher Restore CR
+	# TYPE rancher_restore_info gauge
 	`)
 
 	metrics += rancherRestoreHeader
@@ -58,7 +58,7 @@ func formatRestoreMetadataMetrics(restores []backupv1.Restore) string {
 		}
 
 		metrics += fmt.Sprintf(`
-		rancher_restore{fileName="%s",name="%s",prune="%t",restoreTime="%s",status="%s",storageLocation="%s"} 1
+		rancher_restore_info{fileName="%s",name="%s",prune="%t",restoreTime="%s",status="%s",storageLocation="%s"} 1
 		`, r.Spec.BackupFilename, r.Name, *r.Spec.Prune, r.Status.RestoreCompletionTS, restoreMessage, r.Status.BackupSource)
 	}
 
@@ -291,7 +291,7 @@ var _ = Describe("Restore from remote driver", Ordered, Label("integration"), fu
 				expected := formatRestoreMetadataMetrics(restores.Items)
 
 				return promtestutil.ScrapeAndCompare(metricsURL, strings.NewReader(expected),
-					"rancher_restore",
+					"rancher_restore_info",
 				)
 			}).Should(Succeed())
 		})

pkg/controllers/backup/controller.go

@@ -147,7 +147,7 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error
 		backupStartTS := time.Now()
 		defer func() {
 			backupDoneTS := time.Now()
-			monitoring.UpdateTimeSensitiveBackupMetrics(backup.Name, backupDoneTS.Unix(), backupDoneTS.Sub(backupStartTS).Milliseconds())
+			monitoring.UpdateTimeSensitiveBackupMetrics(backup.Name, float64(backupDoneTS.Unix()), backupDoneTS.Sub(backupStartTS).Seconds())
 			monitoring.UpdateProcessedBackupMetrics(backup.Name, &err)
 		}()
 	}

pkg/monitoring/metrics.go

@@ -21,7 +21,7 @@ import (
 var (
 	backup = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rancher_backup",
+			Name: "rancher_backup_info",
 			Help: "Details on a specific Rancher Backup CR",
 		}, []string{"name", "status", "resourceSetName", "retentionCount", "backupType", "filename", "storageLocation", "nextSnapshot", "lastSnapshot"},
 	)
@@ -35,36 +35,36 @@ var (
 
 	backupsAttempted = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "rancher_backups_attempted",
+			Name: "rancher_backups_attempted_total",
 			Help: "Number of Rancher Backups processed by this operator",
 		}, []string{"name"},
 	)
 
 	backupsFailed = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "rancher_backups_failed",
+			Name: "rancher_backups_failed_total",
 			Help: "Number of failed Rancher Backups processed by this operator",
 		}, []string{"name"},
 	)
 
 	backupDuration = promauto.NewHistogramVec(
 		prometheus.HistogramOpts{
-			Name:    "rancher_backup_duration_ms",
-			Help:    "Duration of each backup processed by this operator in ms",
-			Buckets: []float64{500, 1000, 2500, 5000, 7500, 10000, 30000, 60000, 120000},
+			Name:    "rancher_backup_duration_seconds",
+			Help:    "Duration of each backup processed by this operator in seconds",
+			Buckets: []float64{0.5, 1, 2.5, 5, 7.5, 10, 30, 60, 120},
 		}, []string{"name"},
 	)
 
 	backupLastProcessed = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rancher_backup_last_processed",
+			Name: "rancher_backup_last_processed_timestamp_seconds",
 			Help: "Unix time of when the last Backup was processed (in seconds)",
 		}, []string{"name"},
 	)
 
 	restore = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rancher_restore",
+			Name: "rancher_restore_info",
 			Help: "Details on a specific Rancher Restore CR",
 		}, []string{"name", "status", "fileName", "prune", "storageLocation", "restoreTime"},
 	)
@@ -145,9 +145,9 @@ func UpdateProcessedBackupMetrics(backup string, err *error) {
 	backupsFailed.WithLabelValues(backup)
 }
 
-func UpdateTimeSensitiveBackupMetrics(backup string, endTime int64, totalTime int64) {
-	backupDuration.WithLabelValues(backup).Observe(float64(totalTime))
-	backupLastProcessed.WithLabelValues(backup).Set(float64(endTime))
+func UpdateTimeSensitiveBackupMetrics(backup string, endTime float64, totalTime float64) {
+	backupDuration.WithLabelValues(backup).Observe(totalTime)
+	backupLastProcessed.WithLabelValues(backup).Set(endTime)
 }
 
 func StartRestoreMetricsCollection(