From 4647b6f21cdc3953f94363ce9facea0bc142ec21 Mon Sep 17 00:00:00 2001 From: Dmitriy Alekseev <1865999+dragoangel@users.noreply.github.com> Date: Tue, 15 Oct 2024 02:39:31 +0200 Subject: [PATCH] feat(cluster): Extend monitoring options with relabelings (#358) * feat(cluster): Extend monitoring options --------- Signed-off-by: Dmitriy Alekseev <1865999+dragoangel@users.noreply.github.com> Signed-off-by: Itay Grudev Co-authored-by: Itay Grudev --- charts/cluster/README.md | 8 ++- charts/cluster/templates/cluster.yaml | 19 ++++++ charts/cluster/templates/pooler.yaml | 12 ++++ .../01-monitoring_cluster-assert.yaml | 66 +++++++++++++++++++ .../monitoring/01-monitoring_cluster.yaml | 31 +++++++++ charts/cluster/values.schema.json | 18 +++++ charts/cluster/values.yaml | 22 ++++++- 7 files changed, 174 insertions(+), 2 deletions(-) diff --git a/charts/cluster/README.md b/charts/cluster/README.md index 5b3fcd204..134588a77 100644 --- a/charts/cluster/README.md +++ b/charts/cluster/README.md @@ -160,9 +160,13 @@ refer to the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat | cluster.initdb | object | `{}` | BootstrapInitDB is the configuration of the bootstrap process when initdb is used. See: https://cloudnative-pg.io/documentation/current/bootstrap/ See: https://cloudnative-pg.io/documentation/current/cloudnative-pg.v1/#postgresql-cnpg-io-v1-bootstrapinitdb | | cluster.instances | int | `3` | Number of instances | | cluster.logLevel | string | `"info"` | The instances' log level, one of the following values: error, warning, info (default), debug, trace | -| cluster.monitoring.customQueries | list | `[]` | Custom Prometheus metrics | +| cluster.monitoring.customQueries | list | `[]` | Custom Prometheus metrics Will be stored in the ConfigMap | +| cluster.monitoring.customQueriesSecret | list | `[]` | The list of secrets containing the custom queries | +| cluster.monitoring.disableDefaultQueries | bool | `false` | Whether the default queries should be injected. Set it to true if you don't want to inject default queries into the cluster. | | cluster.monitoring.enabled | bool | `false` | Whether to enable monitoring | | cluster.monitoring.podMonitor.enabled | bool | `true` | Whether to enable the PodMonitor | +| cluster.monitoring.podMonitor.metricRelabelings | list | `[]` | The list of metric relabelings for the PodMonitor. Applied to samples before ingestion. | +| cluster.monitoring.podMonitor.relabelings | list | `[]` | The list of relabelings for the PodMonitor. Applied to samples before scraping. | | cluster.monitoring.prometheusRule.enabled | bool | `true` | Whether to enable the PrometheusRule automated alerts | | cluster.monitoring.prometheusRule.excludeRules | list | `[]` | Exclude specified rules | | cluster.postgresGID | int | `-1` | The GID of the postgres user inside the image, defaults to 26 | @@ -191,6 +195,8 @@ refer to the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat | pooler.instances | int | `3` | Number of PgBouncer instances | | pooler.monitoring.enabled | bool | `false` | Whether to enable monitoring | | pooler.monitoring.podMonitor.enabled | bool | `true` | Whether to enable the PodMonitor | +| pooler.monitoring.podMonitor.metricRelabelings | list | `[]` | The list of metric relabelings for the PodMonitor. Applied to samples before ingestion. | +| pooler.monitoring.podMonitor.relabelings | list | `[]` | The list of relabelings for the PodMonitor. Applied to samples before scraping. | | pooler.parameters | object | `{"default_pool_size":"25","max_client_conn":"1000"}` | PgBouncer configuration parameters | | pooler.poolMode | string | `"transaction"` | PgBouncer pooling mode | | pooler.template | object | `{}` | Custom PgBouncer deployment template. Use to override image, specify resources, etc. | diff --git a/charts/cluster/templates/cluster.yaml b/charts/cluster/templates/cluster.yaml index d5c8bacc4..169683fef 100644 --- a/charts/cluster/templates/cluster.yaml +++ b/charts/cluster/templates/cluster.yaml @@ -76,10 +76,29 @@ spec: monitoring: enablePodMonitor: {{ and .Values.cluster.monitoring.enabled .Values.cluster.monitoring.podMonitor.enabled }} + disableDefaultQueries: {{ .Values.cluster.monitoring.disableDefaultQueries }} {{- if not (empty .Values.cluster.monitoring.customQueries) }} customQueriesConfigMap: - name: {{ include "cluster.fullname" . }}-monitoring key: custom-queries {{- end }} + {{- if not (empty .Values.cluster.monitoring.customQueriesSecret) }} + {{- with .Values.cluster.monitoring.customQueriesSecret }} + customQueriesSecret: + {{- toYaml . | nindent 6 }} + {{ end }} + {{- end }} + {{- if not (empty .Values.cluster.monitoring.podMonitor.relabelings) }} + {{- with .Values.cluster.monitoring.podMonitor.relabelings }} + podMonitorRelabelings: + {{- toYaml . | nindent 6 }} + {{ end }} + {{- end }} + {{- if not (empty .Values.cluster.monitoring.podMonitor.metricRelabelings) }} + {{- with .Values.cluster.monitoring.podMonitor.metricRelabelings }} + podMonitorMetricRelabelings: + {{- toYaml . | nindent 6 }} + {{ end }} + {{- end }} {{ include "cluster.bootstrap" . | nindent 2 }} {{ include "cluster.backup" . | nindent 2 }} diff --git a/charts/cluster/templates/pooler.yaml b/charts/cluster/templates/pooler.yaml index 5e01fe498..e3c7e8402 100644 --- a/charts/cluster/templates/pooler.yaml +++ b/charts/cluster/templates/pooler.yaml @@ -14,6 +14,18 @@ spec: {{- .Values.pooler.parameters | toYaml | nindent 6 }} monitoring: enablePodMonitor: {{ and .Values.pooler.monitoring.enabled .Values.pooler.monitoring.podMonitor.enabled }} + {{- if not (empty .Values.pooler.monitoring.podMonitor.relabelings) }} + {{- with .Values.pooler.monitoring.podMonitor.relabelings }} + podMonitorRelabelings: + {{- toYaml . | nindent 6 }} + {{ end }} + {{- end }} + {{- if not (empty .Values.pooler.monitoring.podMonitor.metricRelabelings) }} + {{- with .Values.pooler.monitoring.podMonitor.metricRelabelings }} + podMonitorMetricRelabelings: + {{- toYaml . | nindent 6 }} + {{ end }} + {{- end }} {{- with .Values.pooler.template }} template: {{- . | toYaml | nindent 4 }} diff --git a/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml b/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml index d69c702c0..2e6b18a44 100644 --- a/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml +++ b/charts/cluster/test/monitoring/01-monitoring_cluster-assert.yaml @@ -1,3 +1,37 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: monitoring-cluster + labels: + foo: bar + annotations: + foo: bar +spec: + instances: 2 + storage: + size: 256Mi + storageClass: standard + monitoring: + disableDefaultQueries: true + customQueriesConfigMap: + - name: monitoring-cluster-monitoring + key: custom-queries + enablePodMonitor: true + podMonitorRelabelings: + - action: replace + replacement: test + targetLabel: environment + - action: replace + replacement: alpha + targetLabel: team + podMonitorMetricRelabelings: + - action: replace + sourceLabels: + - cluster + targetLabel: cnpg_cluster + - action: labeldrop + regex: cluster +--- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: @@ -6,6 +40,22 @@ spec: selector: matchLabels: cnpg.io/cluster: monitoring-cluster + podMetricsEndpoints: + - bearerTokenSecret: + key: '' + name: '' + relabelings: + - targetLabel: environment + replacement: test + - targetLabel: team + replacement: alpha + metricRelabelings: + - action: replace + sourceLabels: + - cluster + targetLabel: cnpg_cluster + - action: labeldrop + regex: cluster --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -15,6 +65,22 @@ spec: selector: matchLabels: cnpg.io/poolerName: monitoring-cluster-pooler-rw + podMetricsEndpoints: + - bearerTokenSecret: + key: '' + name: '' + relabelings: + - targetLabel: environment + replacement: test + - targetLabel: team + replacement: alpha + metricRelabelings: + - action: replace + sourceLabels: + - cluster + targetLabel: cnpg_cluster + - action: labeldrop + regex: cluster --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule diff --git a/charts/cluster/test/monitoring/01-monitoring_cluster.yaml b/charts/cluster/test/monitoring/01-monitoring_cluster.yaml index 2e94c2fc0..f275d4b43 100644 --- a/charts/cluster/test/monitoring/01-monitoring_cluster.yaml +++ b/charts/cluster/test/monitoring/01-monitoring_cluster.yaml @@ -7,6 +7,7 @@ cluster: storageClass: standard monitoring: enabled: true + disableDefaultQueries: true customQueries: - name: "pg_cache_hit_ratio" query: "SELECT current_database() as datname, sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio FROM pg_statio_user_tables;" @@ -17,6 +18,23 @@ cluster: - ratio: usage: GAUGE description: "Cache hit ratio" + podMonitor: + relabelings: + - targetLabel: environment + replacement: test + - targetLabel: team + replacement: alpha + metricRelabelings: + - action: replace + sourceLabels: + - cluster + targetLabel: cnpg_cluster + - action: labeldrop + regex: cluster + additionalLabels: + foo: bar + annotations: + foo: bar backups: enabled: false pooler: @@ -24,3 +42,16 @@ pooler: instances: 1 monitoring: enabled: true + podMonitor: + relabelings: + - targetLabel: environment + replacement: test + - targetLabel: team + replacement: alpha + metricRelabelings: + - action: replace + sourceLabels: + - cluster + targetLabel: cnpg_cluster + - action: labeldrop + regex: cluster diff --git a/charts/cluster/values.schema.json b/charts/cluster/values.schema.json index 9e35d7a90..e20ca872e 100644 --- a/charts/cluster/values.schema.json +++ b/charts/cluster/values.schema.json @@ -214,6 +214,12 @@ "customQueries": { "type": "array" }, + "customQueriesSecret": { + "type": "array" + }, + "disableDefaultQueries": { + "type": "boolean" + }, "enabled": { "type": "boolean" }, @@ -222,6 +228,12 @@ "properties": { "enabled": { "type": "boolean" + }, + "metricRelabelings": { + "type": "array" + }, + "relabelings": { + "type": "array" } } }, @@ -346,6 +358,12 @@ "properties": { "enabled": { "type": "boolean" + }, + "metricRelabelings": { + "type": "array" + }, + "relabelings": { + "type": "array" } } } diff --git a/charts/cluster/values.yaml b/charts/cluster/values.yaml index 8398c74d1..9977f2818 100644 --- a/charts/cluster/values.yaml +++ b/charts/cluster/values.yaml @@ -220,13 +220,23 @@ cluster: podMonitor: # -- Whether to enable the PodMonitor enabled: true + # --The list of relabelings for the PodMonitor. + # Applied to samples before scraping. + relabelings: [] + # -- The list of metric relabelings for the PodMonitor. + # Applied to samples before ingestion. + metricRelabelings: [] prometheusRule: # -- Whether to enable the PrometheusRule automated alerts enabled: true # -- Exclude specified rules excludeRules: [] # - CNPGClusterZoneSpreadWarning + # -- Whether the default queries should be injected. + # Set it to true if you don't want to inject default queries into the cluster. + disableDefaultQueries: false # -- Custom Prometheus metrics + # Will be stored in the ConfigMap customQueries: [] # - name: "pg_cache_hit_ratio" # query: "SELECT current_database() as datname, sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio FROM pg_statio_user_tables;" @@ -237,6 +247,10 @@ cluster: # - ratio: # usage: GAUGE # description: "Cache hit ratio" + # -- The list of secrets containing the custom queries + customQueriesSecret: [] + # - name: custom-queries-secret + # key: custom-queries postgresql: # -- PostgreSQL configuration options (postgresql.conf) @@ -372,8 +386,14 @@ pooler: # -- Whether to enable monitoring enabled: false podMonitor: - # -- Whether to enable the PodMonitor + # -- Whether to enable the PodMonitor enabled: true + # --The list of relabelings for the PodMonitor. + # Applied to samples before scraping. + relabelings: [] + # -- The list of metric relabelings for the PodMonitor. + # Applied to samples before ingestion. + metricRelabelings: [] # -- Custom PgBouncer deployment template. # Use to override image, specify resources, etc.