feat(cluster): Extend monitoring options with relabelings (#358)

* feat(cluster): Extend monitoring options --------- Signed-off-by: Dmitriy Alekseev <[email protected]> Signed-off-by: Itay Grudev <[email protected]> Co-authored-by: Itay Grudev <[email protected]>
cloudnative-pg · Oct 15, 2024 · 4647b6f · 4647b6f
1 parent c7cb7be
commit 4647b6f
Show file tree

Hide file tree

Showing 7 changed files with 174 additions and 2 deletions.
diff --git a/charts/cluster/README.md b/charts/cluster/README.md
@@ -160,9 +160,13 @@ refer to  the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat
 | cluster.initdb | object | `{}` | BootstrapInitDB is the configuration of the bootstrap process when initdb is used. See: https://cloudnative-pg.io/documentation/current/bootstrap/ See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-bootstrapinitdb |
 | cluster.instances | int | `3` | Number of instances |
 | cluster.logLevel | string | `"info"` | The instances' log level, one of the following values: error, warning, info (default), debug, trace |
-| cluster.monitoring.customQueries | list | `[]` | Custom Prometheus metrics |
+| cluster.monitoring.customQueries | list | `[]` | Custom Prometheus metrics Will be stored in the ConfigMap |
+| cluster.monitoring.customQueriesSecret | list | `[]` | The list of secrets containing the custom queries |
+| cluster.monitoring.disableDefaultQueries | bool | `false` | Whether the default queries should be injected. Set it to true if you don't want to inject default queries into the cluster. |
 | cluster.monitoring.enabled | bool | `false` | Whether to enable monitoring |
 | cluster.monitoring.podMonitor.enabled | bool | `true` | Whether to enable the PodMonitor |
+| cluster.monitoring.podMonitor.metricRelabelings | list | `[]` | The list of metric relabelings for the PodMonitor. Applied to samples before ingestion. |
+| cluster.monitoring.podMonitor.relabelings | list | `[]` | The list of relabelings for the PodMonitor. Applied to samples before scraping. |
 | cluster.monitoring.prometheusRule.enabled | bool | `true` | Whether to enable the PrometheusRule automated alerts |
 | cluster.monitoring.prometheusRule.excludeRules | list | `[]` | Exclude specified rules |
 | cluster.postgresGID | int | `-1` | The GID of the postgres user inside the image, defaults to 26 |
@@ -191,6 +195,8 @@ refer to  the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat
 | pooler.instances | int | `3` | Number of PgBouncer instances |
 | pooler.monitoring.enabled | bool | `false` | Whether to enable monitoring |
 | pooler.monitoring.podMonitor.enabled | bool | `true` | Whether to enable the PodMonitor |
+| pooler.monitoring.podMonitor.metricRelabelings | list | `[]` | The list of metric relabelings for the PodMonitor. Applied to samples before ingestion. |
+| pooler.monitoring.podMonitor.relabelings | list | `[]` | The list of relabelings for the PodMonitor. Applied to samples before scraping. |
 | pooler.parameters | object | `{"default_pool_size":"25","max_client_conn":"1000"}` | PgBouncer configuration parameters |
 | pooler.poolMode | string | `"transaction"` | PgBouncer pooling mode |
 | pooler.template | object | `{}` | Custom PgBouncer deployment template. Use to override image, specify resources, etc. |

diff --git a/charts/cluster/templates/cluster.yaml b/charts/cluster/templates/cluster.yaml
@@ -76,10 +76,29 @@ spec:
 
   monitoring:
     enablePodMonitor: {{ and .Values.cluster.monitoring.enabled .Values.cluster.monitoring.podMonitor.enabled }}
+    disableDefaultQueries: {{ .Values.cluster.monitoring.disableDefaultQueries }}
     {{- if not (empty .Values.cluster.monitoring.customQueries) }}
     customQueriesConfigMap:
       - name: {{ include "cluster.fullname" . }}-monitoring
         key: custom-queries
     {{- end }}
+    {{- if not (empty .Values.cluster.monitoring.customQueriesSecret) }}
+    {{- with .Values.cluster.monitoring.customQueriesSecret }}
+    customQueriesSecret:
+      {{- toYaml . | nindent 6 }}
+    {{ end }}
+    {{- end }}
+    {{- if not (empty .Values.cluster.monitoring.podMonitor.relabelings) }}
+    {{- with .Values.cluster.monitoring.podMonitor.relabelings }}
+    podMonitorRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{ end }}
+    {{- end }}
+    {{- if not (empty .Values.cluster.monitoring.podMonitor.metricRelabelings) }}
+    {{- with .Values.cluster.monitoring.podMonitor.metricRelabelings }}
+    podMonitorMetricRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{ end }}
+    {{- end }}
   {{ include "cluster.bootstrap" . | nindent 2 }}
   {{ include "cluster.backup" . | nindent 2 }}
diff --git a/charts/cluster/templates/pooler.yaml b/charts/cluster/templates/pooler.yaml
@@ -14,6 +14,18 @@ spec:
       {{- .Values.pooler.parameters | toYaml | nindent 6 }}
   monitoring:
     enablePodMonitor: {{ and .Values.pooler.monitoring.enabled .Values.pooler.monitoring.podMonitor.enabled }}
+    {{- if not (empty .Values.pooler.monitoring.podMonitor.relabelings) }}
+    {{- with .Values.pooler.monitoring.podMonitor.relabelings }}
+    podMonitorRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{ end }}
+    {{- end }}
+    {{- if not (empty .Values.pooler.monitoring.podMonitor.metricRelabelings) }}
+    {{- with .Values.pooler.monitoring.podMonitor.metricRelabelings }}
+    podMonitorMetricRelabelings:
+      {{- toYaml . | nindent 6 }}
+    {{ end }}
+    {{- end }}
   {{- with .Values.pooler.template }}
   template:
     {{- . | toYaml | nindent 4 }}

diff --git a/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml b/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml
@@ -1,3 +1,37 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: monitoring-cluster
+  labels:
+    foo: bar
+  annotations:
+    foo: bar
+spec:
+  instances: 2
+  storage:
+    size: 256Mi
+    storageClass: standard
+  monitoring:
+      disableDefaultQueries: true
+      customQueriesConfigMap:
+        - name: monitoring-cluster-monitoring
+          key: custom-queries
+      enablePodMonitor: true
+      podMonitorRelabelings:
+        - action: replace
+          replacement: test
+          targetLabel: environment
+        - action: replace
+          replacement: alpha
+          targetLabel: team
+      podMonitorMetricRelabelings:
+        - action: replace
+          sourceLabels:
+            - cluster
+          targetLabel: cnpg_cluster
+        - action: labeldrop
+          regex: cluster
+---
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
 metadata:
@@ -6,6 +40,22 @@ spec:
   selector:
     matchLabels:
       cnpg.io/cluster: monitoring-cluster
+  podMetricsEndpoints:
+    - bearerTokenSecret:
+        key: ''
+        name: ''
+      relabelings:
+        - targetLabel: environment
+          replacement: test
+        - targetLabel: team
+          replacement: alpha
+      metricRelabelings:
+        - action: replace
+          sourceLabels:
+            - cluster
+          targetLabel: cnpg_cluster
+        - action: labeldrop
+          regex: cluster
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PodMonitor
@@ -15,6 +65,22 @@ spec:
   selector:
     matchLabels:
       cnpg.io/poolerName: monitoring-cluster-pooler-rw
+  podMetricsEndpoints:
+    - bearerTokenSecret:
+        key: ''
+        name: ''
+      relabelings:
+        - targetLabel: environment
+          replacement: test
+        - targetLabel: team
+          replacement: alpha
+      metricRelabelings:
+        - action: replace
+          sourceLabels:
+            - cluster
+          targetLabel: cnpg_cluster
+        - action: labeldrop
+          regex: cluster
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule

diff --git a/charts/cluster/test/monitoring/01-monitoring_cluster.yaml b/charts/cluster/test/monitoring/01-monitoring_cluster.yaml
@@ -7,6 +7,7 @@ cluster:
     storageClass: standard
   monitoring:
     enabled: true
+    disableDefaultQueries: true
     customQueries:
       - name: "pg_cache_hit_ratio"
         query: "SELECT current_database() as datname, sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio FROM pg_statio_user_tables;"
@@ -17,10 +18,40 @@ cluster:
           - ratio:
               usage: GAUGE
               description: "Cache hit ratio"
+    podMonitor:
+      relabelings:
+        - targetLabel: environment
+          replacement: test
+        - targetLabel: team
+          replacement: alpha
+      metricRelabelings:
+        - action: replace
+          sourceLabels:
+            - cluster
+          targetLabel: cnpg_cluster
+        - action: labeldrop
+          regex: cluster
+  additionalLabels:
+    foo: bar
+  annotations:
+    foo: bar
 backups:
   enabled: false
 pooler:
   enabled: true
   instances: 1
   monitoring:
     enabled: true
+    podMonitor:
+      relabelings:
+        - targetLabel: environment
+          replacement: test
+        - targetLabel: team
+          replacement: alpha
+      metricRelabelings:
+        - action: replace
+          sourceLabels:
+            - cluster
+          targetLabel: cnpg_cluster
+        - action: labeldrop
+          regex: cluster
diff --git a/charts/cluster/values.schema.json b/charts/cluster/values.schema.json
@@ -214,6 +214,12 @@
                         "customQueries": {
                             "type": "array"
                         },
+                        "customQueriesSecret": {
+                            "type": "array"
+                        },
+                        "disableDefaultQueries": {
+                            "type": "boolean"
+                        },
                         "enabled": {
                             "type": "boolean"
                         },
@@ -222,6 +228,12 @@
                             "properties": {
                                 "enabled": {
                                     "type": "boolean"
+                                },
+                                "metricRelabelings": {
+                                    "type": "array"
+                                },
+                                "relabelings": {
+                                    "type": "array"
                                 }
                             }
                         },
@@ -346,6 +358,12 @@
                             "properties": {
                                 "enabled": {
                                     "type": "boolean"
+                                },
+                                "metricRelabelings": {
+                                    "type": "array"
+                                },
+                                "relabelings": {
+                                    "type": "array"
                                 }
                             }
                         }

diff --git a/charts/cluster/values.yaml b/charts/cluster/values.yaml
@@ -220,13 +220,23 @@ cluster:
     podMonitor:
       # -- Whether to enable the PodMonitor
       enabled: true
+      # --The list of relabelings for the PodMonitor.
+      # Applied to samples before scraping.
+      relabelings: []
+      # -- The list of metric relabelings for the PodMonitor.
+      # Applied to samples before ingestion.
+      metricRelabelings: []
     prometheusRule:
       # -- Whether to enable the PrometheusRule automated alerts
       enabled: true
       # -- Exclude specified rules
       excludeRules: []
         # - CNPGClusterZoneSpreadWarning
+    # -- Whether the default queries should be injected.
+    # Set it to true if you don't want to inject default queries into the cluster.
+    disableDefaultQueries: false
     # -- Custom Prometheus metrics
+    # Will be stored in the ConfigMap
     customQueries: []
     #  - name: "pg_cache_hit_ratio"
     #    query: "SELECT current_database() as datname, sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio FROM pg_statio_user_tables;"
@@ -237,6 +247,10 @@ cluster:
     #      - ratio:
     #          usage: GAUGE
     #          description: "Cache hit ratio"
+    # -- The list of secrets containing the custom queries
+    customQueriesSecret: []
+    #  - name: custom-queries-secret
+    #    key: custom-queries
 
   postgresql:
     # -- PostgreSQL configuration options (postgresql.conf)
@@ -372,8 +386,14 @@ pooler:
     # -- Whether to enable monitoring
     enabled: false
     podMonitor:
-        # -- Whether to enable the PodMonitor
+      # -- Whether to enable the PodMonitor
       enabled: true
+      # --The list of relabelings for the PodMonitor.
+      # Applied to samples before scraping.
+      relabelings: []
+      # -- The list of metric relabelings for the PodMonitor.
+      # Applied to samples before ingestion.
+      metricRelabelings: []
 
   # -- Custom PgBouncer deployment template.
   # Use to override image, specify resources, etc.