feat ✨: add monitoring

Signed-off-by: Victor Hang <[email protected]>
Bealvio · Feb 28, 2025 · 85bdc9e · 85bdc9e
1 parent 7481aa8
commit 85bdc9e
Show file tree

Hide file tree

Showing 156 changed files with 110,685 additions and 1 deletion.
diff --git a/gitops/apps/crossplane/resources/kustomization.yaml b/gitops/apps/crossplane/resources/kustomization.yaml
@@ -1,6 +1,6 @@
 namespace: crossplane-system
 resources:
   - instancegithub.yaml
-  # - applications.yaml
+  - applications.yaml
   - org.yaml
   - project.yaml
diff --git a/gitops/apps/monitoring/alertmanager-config.yaml b/gitops/apps/monitoring/alertmanager-config.yaml
@@ -0,0 +1,80 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: main
+spec:
+  inhibitRules:
+    - sourceMatch:
+        - matchType: '='
+          name: 'severity'
+          value: 'critical'
+      targetMatch:
+        - matchType: '=~'
+          name: 'severity'
+          value: 'warning|info|none'
+      equal:
+        - namespace
+        - alertname
+        - name
+    - sourceMatch:
+        - matchType: '='
+          name: 'severity'
+          value: 'warning'
+      targetMatch:
+        - matchType: '=~'
+          name: 'severity'
+          value: 'info|none'
+      equal:
+        - namespace
+        - alertname
+        - name
+    - sourceMatch:
+        - matchType: '='
+          name: 'alertname'
+          value: 'InfoInhibitor'
+      targetMatch:
+        - matchType: '='
+          name: 'severity'
+          value: 'info'
+      equal:
+        - namespace
+  route:
+    repeatInterval: 12h
+    receiver: discord
+    groupBy:
+      - alertname
+      - namespace
+      - exported_namespace
+      - name
+    routes:
+      - matchers:
+          - matchType: '=~'
+            name: 'alertname'
+            value: 'InfoInhibitor|Watchdog|KubeCPUOvercommit|KubeMemoryOvercommit|CDIDefaultStorageClassDegraded|KubeVirtNoAvailableNodesToRunVMs|LowKVMNodesCount|KubeVirtDeprecatedAPIRequested'
+        receiver: 'null'
+  receivers:
+    - name: 'null'
+    - name: discord
+      discordConfigs:
+        - sendResolved: true
+          apiURL:
+            name: grafana-secrets
+            key: discordwebhookurl
+          title: |-
+            **[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ if ne .CommonAnnotations.summary ""}}{{ .CommonAnnotations.summary }}{{ else }}{{ .CommonLabels.alertname }}{{ end }}**
+          message: |-
+            {{ range .Alerts }}
+            ---
+            **🚨 Alert:** {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity | toUpper }}`{{ end }}
+
+            **📄 Description:**
+            ```
+            {{ if ne .Annotations.description ""}}{{ .Annotations.description }}{{ else }}N/A{{ end }}
+            ```
+
+            **🔍 Details:**
+            {{ range .Labels.SortedPairs }}
+            **{{ .Name }}**: `{{ .Value }}`
+            {{ end }}
+            ---
+            {{ end }}
diff --git a/gitops/apps/monitoring/alertmanager-patch.yaml b/gitops/apps/monitoring/alertmanager-patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: monitoring.coreos.com/v1
+kind: Alertmanager
+metadata:
+  name: main
+  namespace: monitoring
+spec:
+  alertmanagerConfiguration:
+    name: main
diff --git a/gitops/apps/monitoring/blackbox-exporter-cm-patch.yaml b/gitops/apps/monitoring/blackbox-exporter-cm-patch.yaml
@@ -0,0 +1,60 @@
+apiVersion: v1
+data:
+  config.yml: |-
+    "modules":
+      "http_2xx":
+        "http":
+          "preferred_ip_protocol": "ip4"
+        "prober": "http"
+      "http_2xx_didactiklabs":
+        "http":
+          "preferred_ip_protocol": "ip4"
+          "tls_config":
+            "ca_file": "/bealv/ca.crt"
+            "insecure_skip_verify": false
+        "prober": "http"
+      "http_post_2xx":
+        "http":
+          "method": "POST"
+          "preferred_ip_protocol": "ip4"
+        "prober": "http"
+      "irc_banner":
+        "prober": "tcp"
+        "tcp":
+          "preferred_ip_protocol": "ip4"
+          "query_response":
+          - "send": "NICK prober"
+          - "send": "USER prober prober prober :prober"
+          - "expect": "PING :([^ ]+)"
+            "send": "PONG ${1}"
+          - "expect": "^:[^ ]+ 001"
+      "pop3s_banner":
+        "prober": "tcp"
+        "tcp":
+          "preferred_ip_protocol": "ip4"
+          "query_response":
+          - "expect": "^+OK"
+          "tls": true
+          "tls_config":
+            "insecure_skip_verify": false
+      "ssh_banner":
+        "prober": "tcp"
+        "tcp":
+          "preferred_ip_protocol": "ip4"
+          "query_response":
+          - "expect": "^SSH-2.0-"
+      "tcp_connect":
+        "prober": "tcp"
+        "tcp":
+          "preferred_ip_protocol": "ip4"
+      "http_2xx_with_sa_auth":
+        "prober": "http"
+        "http":
+          "preferred_ip_protocol": "ip4"
+          "tls_config":
+            "ca_file": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+            "insecure_skip_verify": false
+          "bearer_token_file": "/var/run/secrets/kubernetes.io/serviceaccount/token"
+kind: ConfigMap
+metadata:
+  name: blackbox-exporter-configuration
diff --git a/gitops/apps/monitoring/blackbox-exporter-patch.yaml b/gitops/apps/monitoring/blackbox-exporter-patch.yaml
@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: blackbox-exporter
+  namespace: monitoring
+spec:
+  template:
+    spec:
+      volumes:
+        - name: ca-cert-volume
+          secret:
+            secretName: bealv-mgmt-bundle
+            items:
+              - key: ca.crt
+                path: ca.crt
+      containers:
+        - name: blackbox-exporter
+          volumeMounts:
+            - name: ca-cert-volume
+              mountPath: /bealv/ca.crt
+              subPath: ca.crt
+              readOnly: true
diff --git a/gitops/apps/monitoring/grafana-alerts/kustomization.yaml b/gitops/apps/monitoring/grafana-alerts/kustomization.yaml
@@ -0,0 +1 @@
+resources: []
diff --git a/gitops/apps/monitoring/grafana-contactpoints.yaml b/gitops/apps/monitoring/grafana-contactpoints.yaml
@@ -0,0 +1,17 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaContactPoint
+metadata:
+  name: discord
+spec:
+  name: discord
+  type: 'discord'
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  settings:
+  valuesFrom:
+    - targetPath: url
+      valueFrom:
+        secretKeyRef:
+          name: grafana-secrets
+          key: discordwebhookurl
diff --git a/gitops/apps/monitoring/grafana-dashboards/administration-dashboards.yaml b/gitops/apps/monitoring/grafana-dashboards/administration-dashboards.yaml
@@ -0,0 +1,92 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: node-exporter
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  grafanaCom:
+    id: 1860
+    revision: 37
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: cluster-overview
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  grafanaCom:
+    id: 21410
+    revision: 3
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: flux-cluster
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  url: https://raw.githubusercontent.com/fluxcd/flux2-monitoring-example/refs/heads/main/monitoring/configs/dashboards/cluster.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: contour
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  grafanaCom:
+    id: 21396
+    revision: 2
+  datasources:
+    - datasourceName: prometheus
+      inputName: 'DS_DEVELOPERS'
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: cert-manager
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  grafanaCom:
+    id: 20842
+    revision: 3
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: coredns
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  url: https://raw.githubusercontent.com/monitoring-mixins/website/refs/heads/master/assets/coredns/dashboards/coredns.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: blackbox-exporter
+spec:
+  folder: administration
+  instanceSelector:
+    matchLabels:
+      dashboards: 'grafana'
+  grafanaCom:
+    id: 13659
+    revision: 1
+  datasources:
+    - datasourceName: prometheus
+      inputName: 'DS_PROMETHEUS'