chore(deps): update helm release victoria-metrics-k8s-stack to v0.27.2

[renovate]

This PR contains the following updates:

Package	Update	Change
victoria-metrics-k8s-stack	patch	0.27.0 -> 0.27.2

---

Release Notes

VictoriaMetrics/helm-charts (victoria-metrics-k8s-stack)

v0.27.2

Compare Source

Release notes for version 0.27.2

Release date: 2023-10-04

• bump version of VM operator to 0.39.0

v0.27.1

Compare Source

Release notes for version 0.27.1

Release date: 2023-09-28

• Fix relabelConfigs for operator's VMServiceScrape (#​624)

---

Configuration

📅 Schedule: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).

🚦 Automerge: Enabled.

♻ Rebasing: Whenever PR is behind base branch, or you tick the rebase/retry checkbox.

🔕 Ignore: Close this PR and you won't be reminded about this update again.

---

• If you want to rebase/retry this PR, check this box

---

This PR was generated by Mend Renovate. View the repository job log.

[github-actions]

--- kubernetes/apps/monitoring/victoria-metrics/app Kustomization: flux-system/victoria-metrics HelmRelease: monitoring/victoria-metrics

+++ kubernetes/apps/monitoring/victoria-metrics/app Kustomization: flux-system/victoria-metrics HelmRelease: monitoring/victoria-metrics

@@ -14,13 +14,13 @@

       chart: victoria-metrics-k8s-stack
       interval: 15m
       sourceRef:
         kind: HelmRepository
         name: victoria-metrics
         namespace: flux-system
-      version: 0.27.0
+      version: 0.27.2
   install:
     createNamespace: true
     remediation:
       retries: 3
   interval: 5m
   maxHistory: 2
@@ -31,17 +31,18 @@

     crds: CreateReplace
     remediation:
       retries: 3
   values:
     alertmanager:
       enabled: false
+    defaultDashboards:
+      grafanaOperator:
+        enabled: true
     fullnameOverride: vmetrics
     grafana:
       enabled: false
-    grafanaOperatorDashboardsFormat:
-      enabled: true
     kubeEtcd:
       enabled: false
     vmalert:
       enabled: false
     vmsingle:
       spec:

[github-actions]

--- HelmRelease: monitoring/victoria-metrics Deployment: monitoring/victoria-metrics-victoria-metrics-operator

+++ HelmRelease: monitoring/victoria-metrics Deployment: monitoring/victoria-metrics-victoria-metrics-operator

@@ -17,12 +17,13 @@

   template:
     metadata:
       labels:
         app.kubernetes.io/name: victoria-metrics-operator
         app.kubernetes.io/instance: victoria-metrics
     spec:
+      securityContext: {}
       serviceAccountName: victoria-metrics-victoria-metrics-operator
       containers:
       - name: operator
         image: victoriametrics/operator:v0.48.3
         imagePullPolicy: IfNotPresent
         env:
@@ -67,12 +68,14 @@

           timeoutSeconds: 5
         volumeMounts:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
         resources: {}
+        securityContext: {}
+      terminationGracePeriodSeconds: 30
       volumes:
       - name: cert
         secret:
           defaultMode: 420
           secretName: victoria-metrics-victoria-metrics-operator-validation
 
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-alertmanager-overview

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-alertmanager-overview

@@ -1,18 +0,0 @@

----
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  namespace: monitoring
-  name: vmetrics-alertmanager-overview
-  labels:
-    app: victoria-metrics-k8s-stack-grafana
-    app.kubernetes.io/instance: victoria-metrics
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/name: victoria-metrics-k8s-stack
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: grafana
-  json: |
-    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":false,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"current set of alerts stored in the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid alerts received by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Received","refId":"A"},{"expr":"sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Invalid","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts receive rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Alerts","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Total","refId":"A"},{"expr":"sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Failed","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notifications Send Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"latency of notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99,\n  sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} 99th Percentile","refId":"A"},{"expr":"histogram_quantile(0.50,\n  sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Median","refId":"B"},{"expr":"sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notification Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Notifications","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["alertmanager-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"text":"Prometheus","value":"Prometheus"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"namespace","multi":false,"name":"namespace","options":[],"query":"label_values(alertmanager_alerts, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"service","multi":false,"name":"service","options":[],"query":"label_values(alertmanager_alerts, service)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"all","value":"$__all"},"datasource":"$datasource","hide":2,"includeAll":true,"label":null,"multi":false,"name":"integration","options":[],"query":"label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"utc","title":"Alertmanager / Overview","uid":"alertmanager-overview","version":0}
-
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-grafana-overview

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-grafana-overview

@@ -1,18 +0,0 @@

----
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  namespace: monitoring
-  name: vmetrics-grafana-overview
-  labels:
-    app: victoria-metrics-k8s-stack-grafana
-    app.kubernetes.io/instance: victoria-metrics
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/name: victoria-metrics-k8s-stack
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: grafana
-  json: |
-    {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":false,"gnetId":null,"graphTooltip":0,"id":3085,"iteration":1631554945276,"links":[],"panels":[{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":0,"y":0},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Firing Alerts","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":6,"y":0},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})","interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Dashboards","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{"align":null,"displayMode":"auto"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":0},"id":10,"options":{"showHeader":true},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_build_info{job=~\"$job\", instance=~\"$instance\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Build Info","transformations":[{"id":"labelsToFields","options":{}},{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"branch":true,"container":true,"goversion":true,"namespace":true,"pod":true,"revision":true},"indexByName":{"Time":7,"Value":11,"branch":4,"container":8,"edition":2,"goversion":6,"instance":1,"job":0,"namespace":9,"pod":10,"revision":5,"version":3},"renameByName":{}}}],"type":"table"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":5},"hiddenSeries":false,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m]))","interval":"","legendFormat":"{{status_code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"RPS","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:157","format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:158","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":5},"hiddenSeries":false,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"exemplar":true,"expr":"histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"99th Percentile","refId":"A"},{"exemplar":true,"expr":"histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"50th Percentile","refId":"B"},{"exemplar":true,"expr":"sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))","interval":"","legendFormat":"Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Request Latency","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:210","format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:211","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"schemaVersion":30,"style":"dark","tags":["vm-k8s-stack"],"templating":{"list":[{"current":{"selected":true,"text":"dev-cortex","value":"dev-cortex"},"description":null,"error":null,"hide":0,"includeAll":false,"label":null,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".*","current":{"selected":false,"text":["default/grafana"],"value":["default/grafana"]},"datasource":"$datasource","definition":"label_values(grafana_build_info, job)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"job","options":[],"query":{"query":"label_values(grafana_build_info, job)","refId":"Billing Admin-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".*","current":{"selected":false,"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(grafana_build_info, instance)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(grafana_build_info, instance)","refId":"Billing Admin-instance-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"Grafana Overview","uid":"6be0s85Mk","version":2}
-
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-system-coredns

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-system-coredns

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-global

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-global

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-namespaces

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-namespaces

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-pods

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-pods

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-node-exporter-full

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-node-exporter-full

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-backupmanager

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-backupmanager

@@ -1,18 +0,0 @@

----
-apiVersion: grafana.integreatly.org/v1beta1
-kind: GrafanaDashboard
-metadata:
-  namespace: monitoring
-  name: vmetrics-victoriametrics-backupmanager
-  labels:
-    app: victoria-metrics-k8s-stack-grafana
-    app.kubernetes.io/instance: victoria-metrics
-    app.kubernetes.io/managed-by: Helm
-    app.kubernetes.io/name: victoria-metrics-k8s-stack
-spec:
-  instanceSelector:
-    matchLabels:
-      dashboards: grafana
-  json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-operator

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-operator

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-single-node

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-single-node

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-vmagent

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-victoriametrics-vmagent

@@ -10,9 +10,10 @@

     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: victoria-metrics-k8s-stack
 spec:
   instanceSelector:
     matchLabels:
       dashboards: grafana
+  allowCrossNamespaceImport: false
   json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kube-apiserver-slos

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kube-apiserver-slos

@@ -17,57 +17,57 @@

     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
-        and
-        sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate1h) > (14.40 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate5m) > (14.40 * 0.01000)
       for: 2m
       labels:
         long: 1h
         severity: critical
         short: 5m
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate6h) > (6.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate30m) > (6.00 * 0.01000)
       for: 15m
       labels:
         long: 6h
         severity: critical
         short: 30m
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate1d) > (3.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate2h) > (3.00 * 0.01000)
       for: 1h
       labels:
         long: 1d
         severity: warning
         short: 2h
     - alert: KubeAPIErrorBudgetBurn
       annotations:
         description: The API server is burning too much error budget.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
         summary: The API server is burning too much error budget.
       expr: |-
-        sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
-        and
-        sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
+        sum by (cluster) (apiserver_request:burnrate3d) > (1.00 * 0.01000)
+        and on (cluster)
+        sum by (cluster) (apiserver_request:burnrate6h) > (1.00 * 0.01000)
       for: 3h
       labels:
         long: 3d
         severity: warning
         short: 6h
 
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kubernetes-apps

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kubernetes-apps

@@ -125,13 +125,13 @@

         description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }}
           update has not been rolled out.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
         summary: StatefulSet update has not been rolled out.
       expr: |-
         (
-          max by (namespace,statefulset,cluster) (
+          max by (namespace,statefulset,job,cluster) (
             kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
               unless
             kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
           )
             *
           (
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kubernetes-system-apiserver

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-kubernetes-system-apiserver

@@ -75,13 +75,14 @@

       annotations:
         description: The kubernetes apiserver has terminated {{ $value | humanizePercentage
           }} of its incoming requests.
         runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
         summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage
           }} of its incoming requests.
-      expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))  /
-        (  sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
+      expr: sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
+        / ( sum by (cluster) (rate(apiserver_request_total{job="apiserver"}[10m]))
+        + sum by (cluster) (rate(apiserver_request_terminations_total{job="apiserver"}[10m]))
         ) > 0.20
       for: 5m
       labels:
         severity: warning
 
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-vm-health

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-vm-health

@@ -62,12 +62,23 @@

         description: |
           Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process.
         summary: More than 90% of CPU is used by "{{ $labels.job }}"("{{ $labels.instance
           }}") during the last 5m
       expr: rate(process_cpu_seconds_total[5m]) / process_cpu_cores_available > 0.9
       for: 5m
+      labels:
+        severity: critical
+    - alert: TooHighGoroutineSchedulingLatency
+      annotations:
+        description: |
+          Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.
+        summary: '"{{ $labels.job }}"("{{ $labels.instance }}") has insufficient CPU
+          resources for >15m'
+      expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket[5m]))
+        by (le,job,instance,cluster)) > 0.1
+      for: 15m
       labels:
         severity: critical
     - alert: TooManyLogs
       annotations:
         description: |
           Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m. Worth to check logs for specific error messages.
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-vmagent

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-vmagent

@@ -30,36 +30,39 @@

       labels:
         severity: critical
     - alert: RejectedRemoteWriteDataBlocksAreDropped
       annotations:
         dashboard: grafana.domain.com/d/G7Z9GzMGz?viewPanel=79&var-instance={{ $labels.instance
           }}
-        summary: Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops
+        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops
           the rejected by remote-write server data blocks. Check the logs to find
           the reason for rejects.
+        summary: Vmagent is dropping data blocks that are rejected by remote storage
       expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) without (url)
         > 0
       for: 15m
       labels:
         severity: warning
     - alert: TooManyScrapeErrors
       annotations:
         dashboard: grafana.domain.com/d/G7Z9GzMGz?viewPanel=31&var-instance={{ $labels.instance
           }}
-        summary: Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails
+        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails
           to scrape targets for last 15m
+        summary: Vmagent fails to scrape one or more targets
       expr: increase(vm_promscrape_scrapes_failed_total[5m]) > 0
       for: 15m
       labels:
         severity: warning
     - alert: TooManyWriteErrors
       annotations:
         dashboard: grafana.domain.com/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance
           }}
-        summary: Job "{{ $labels.job }}" on instance {{ $labels.instance }} responds
+        description: Job "{{ $labels.job }}" on instance {{ $labels.instance }} responds
           with errors to write requests for last 15m.
+        summary: Vmagent responds with too many errors on data ingestion protocols
       expr: |-
         (sum(increase(vm_ingestserver_request_errors_total[5m])) without (name,net,type)
         +
         sum(increase(vmagent_http_request_errors_total[5m])) without (path,protocol)) > 0
       for: 15m
       labels:
--- HelmRelease: monitoring/victoria-metrics ValidatingWebhookConfiguration: monitoring/victoria-metrics-victoria-metrics-operator-admission

+++ HelmRelease: monitoring/victoria-metrics ValidatingWebhookConfiguration: monitoring/victoria-metrics-victoria-metrics-operator-admission

@@ -11,13 +11,13 @@

 - clientConfig:
     service:
       namespace: monitoring
       name: victoria-metrics-victoria-metrics-operator
       path: /validate-operator-victoriametrics-com-v1beta1-vlogs
       port: 9443
-    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUsrY0d0ckhFNWtzKzA5ZGNjZ2hDaE13RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qUXhNREV3TURreU5URTRXaGNOTXpReApNREE0TURreU5URTRXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNUL2JXRStyREVnWUpiTGx2STUwQUE2YUUwK2JBVHBZWXoKOFRBK3lZenk2ZXhqUnVyUFVycXJqTWNGNEllbGpTbWNIMUZNcFM3c2s1Wlk0MmhhRnFCN29DM0RiUFBBRCtvTQpxcXNwR21tSGk5R1FVM3RxWWw5Tm9IcDkyTjJ2S2FXRXJVcHZacmk0V0lqUHpWQitHUGJPZzNLR2FHMVhjUkJVClI1a3gyeU1qMmh4MWtzUy96VERSamtldFBDU2daU05mWUI0dUc5SzhtVWZQa0ZLRTJVVnFCOTZVd3RMUWE4OEgKeFNQWnQxeFVHTFp1ay85bHRIbDFpZkJVL2pKY0VUNDdRRCtJR3VLUGpNM0hia1VqMHZOUU9RODdObWFiQ0t5UgpqMVNvQmhlN3ZSazRFM2lZTDBZaEhkLzZTZ3hjYkFKUGVYY05vU2N1NXpnWjBMTFZlRmtDQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlROWG9mWCtqTkk1L1JGMlZ0dkJndkZmcFN0SURBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFmQzYrcll3NFRKdW5KQklNTmFWRnZEdW1aUnc5UyttWE9lcU5wNEVPCnprK1k1V1F4K2luM0R6OTFBdENsMWRsRWpsdGQyeDJ4ajg4SjlqQ3JJaWVLeVJsVW5CNWdMVVp2RnFpbkx4VU0KelBZVTR0S3lXM2lDaUp0TnB4VGVRMXRVZUg4OWpYdDdrS05LYTUzb2NMWVRSWjJ3K2xXKzVnZVBZUGFhZkp4dApNMTgzRDFSWVM3TlZVZEhSYVVBRkRuSWRYOVJPMHQ3UUJSdm1KTlZSbE1GNENieW9iTzVXZUwvZ0ZMZGplaUNTCmYzZitTUXc5U1FEVFBVcllId2Z6ak90aUpWVGdaR1NiZjcvYXo5MUpvSElkSzVDdExDT2hxL1BqODNRSkxCRE4Kb1JoWjZFSzEzdVpyNFI3Yld2Kzk4MDBlYi9CUUpPc0J6RTd2ek9yeFBrVjIrUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIVENDQWdXZ0F3SUJBZ0lRZk9DNHJLUXdQMDdocnpObXFReFgxekFOQmdrcWhraUc5dzBCQVFzRkFEQVoKTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVEFlRncweU5ERXdNVEF3T1RJMU1UaGFGdzB6TkRFdwpNRGd3T1RJMU1UaGFNQmt4RnpBVkJnTlZCQU1URG5adExXOXdaWEpoZEc5eUxXTmhNSUlCSWpBTkJna3Foa2lHCjl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF5NWNVRVV1V0Z0c1lWRnAwVFdWZ3diR1ArdndUdTlUMHJrTi8KL3hLQ0FTaEwrTlZ0V1dWTW80Q3pGSWNpaUFiRXZsL2FvK1NQQU0rNkg0OTBiVlg5OTBhU2p2Um1CTlFReXZkVAoyVTQ2bTRDb3BNZ29UeXlFdi8xWm5SbWhSeGU1dnBFSFZsN1NWcGJFcWtxM3RqT3JXall3d0xFcXBwOXprQ21ICjdOaWptYzlhck1ERG5DbnR0SU95ekdPT0kzMGEwUjNjRGl6Rjc0aWtKZmYrZ2l0YlRaU1ZEL0RTWWp5UmJZa0sKTzJqMTJUTytJb0JObnNWRlAvVXRvY2JrNU1SWUxBb2c2aWJaMWJsTVdHSWl5TENaNGNlV1lUcGVJNVJTL2lzbgpQVllvck1rMzZicjN0WFdvcHJaVzZwT1NKeUNuTzA4VXZ3QmdIZ0VMOHJWYUVjdWhiUUlEQVFBQm8yRXdYekFPCkJnTlZIUThCQWY4RUJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEcKQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGR0RLanJXNmJlT2RnS0QvYkFKUlRwTWhNc1hvTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ1F3YUMycWVYSUxST1ZaTE9JTFAxNGh2WW55SElCMkdvQ2VpSHVvY0JOCll5dVhzamV4NWEzNDVYRGdGK1FsT0RpdnJ1WjlPMVdkUTk2UmZ2ZWQ3QnA3M2U3aHYrRVVmQUhJQlM4dXRIUzQKYVNBRjN0R3BoRTM2ck9KUzMwcmpBbkg5bzh0dWsxbVA1U2xOay9XUXlJdnc0cDROVFg2VzhRR3FQU01jL09INgpxVlF1YzlEdTlKcEFZYVVWOHBjZUhhK3BGVzlpUm9XL1Y5TDZINlBSREFyMWQ5OHA2MlBhdS9wbXV6Ni9kVW4zCmFvN0FnZHc3UTZRSnRsZ1g4NEdiblVadnZXVFdjZ05wZzZUcFJvdGx1SjBSbDlXZ1dDK0VxTjdQR3BmLzRWaTEKL3ZYRkQ0YWF0WWNTSUNieWtSejliRXlBeVNORHZxU0NTK0dPRmdrU29QWk4KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
   failurePolicy: Fail
   name: vlogs.victoriametrics.com
   admissionReviewVersions:
   - v1
   - v1beta1
   sideEffects: None
@@ -40,13 +40,13 @@

 - clientConfig:
     service:
       namespace: monitoring
       name: victoria-metrics-victoria-metrics-operator
       path: /validate-operator-victoriametrics-com-v1beta1-vmagent
       port: 9443
-    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUsrY0d0ckhFNWtzKzA5ZGNjZ2hDaE13RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qUXhNREV3TURreU5URTRXaGNOTXpReApNREE0TURreU5URTRXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNUL2JXRStyREVnWUpiTGx2STUwQUE2YUUwK2JBVHBZWXoKOFRBK3lZenk2ZXhqUnVyUFVycXJqTWNGNEllbGpTbWNIMUZNcFM3c2s1Wlk0MmhhRnFCN29DM0RiUFBBRCtvTQpxcXNwR21tSGk5R1FVM3RxWWw5Tm9IcDkyTjJ2S2FXRXJVcHZacmk0V0lqUHpWQitHUGJPZzNLR2FHMVhjUkJVClI1a3gyeU1qMmh4MWtzUy96VERSamtldFBDU2daU05mWUI0dUc5SzhtVWZQa0ZLRTJVVnFCOTZVd3RMUWE4OEgKeFNQWnQxeFVHTFp1ay85bHRIbDFpZkJVL2pKY0VUNDdRRCtJR3VLUGpNM0hia1VqMHZOUU9RODdObWFiQ0t5UgpqMVNvQmhlN3ZSazRFM2lZTDBZaEhkLzZTZ3hjYkFKUGVYY05vU2N1NXpnWjBMTFZlRmtDQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlROWG9mWCtqTkk1L1JGMlZ0dkJndkZmcFN0SURBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFmQzYrcll3NFRKdW5KQklNTmFWRnZEdW1aUnc5UyttWE9lcU5wNEVPCnprK1k1V1F4K2luM0R6OTFBdENsMWRsRWpsdGQyeDJ4ajg4SjlqQ3JJaWVLeVJsVW5CNWdMVVp2RnFpbkx4VU0KelBZVTR0S3lXM2lDaUp0TnB4VGVRMXRVZUg4OWpYdDdrS05LYTUzb2NMWVRSWjJ3K2xXKzVnZVBZUGFhZkp4dApNMTgzRDFSWVM3TlZVZEhSYVVBRkRuSWRYOVJPMHQ3UUJSdm1KTlZSbE1GNENieW9iTzVXZUwvZ0ZMZGplaUNTCmYzZitTUXc5U1FEVFBVcllId2Z6ak90aUpWVGdaR1NiZjcvYXo5MUpvSElkSzVDdExDT2hxL1BqODNRSkxCRE4Kb1JoWjZFSzEzdVpyNFI3Yld2Kzk4MDBlYi9CUUpPc0J6RTd2ek9yeFBrVjIrUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIVENDQWdXZ0F3SUJBZ0lRZk9DNHJLUXdQMDdocnpObXFReFgxekFOQmdrcWhraUc5dzBCQVFzRkFEQVoKTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVEFlRncweU5ERXdNVEF3T1RJMU1UaGFGdzB6TkRFdwpNRGd3T1RJMU1UaGFNQmt4RnpBVkJnTlZCQU1URG5adExXOXdaWEpoZEc5eUxXTmhNSUlCSWpBTkJna3Foa2lHCjl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF5NWNVRVV1V0Z0c1lWRnAwVFdWZ3diR1ArdndUdTlUMHJrTi8KL3hLQ0FTaEwrTlZ0V1dWTW80Q3pGSWNpaUFiRXZsL2FvK1NQQU0rNkg0OTBiVlg5OTBhU2p2Um1CTlFReXZkVAoyVTQ2bTRDb3BNZ29UeXlFdi8xWm5SbWhSeGU1dnBFSFZsN1NWcGJFcWtxM3RqT3JXall3d0xFcXBwOXprQ21ICjdOaWptYzlhck1ERG5DbnR0SU95ekdPT0kzMGEwUjNjRGl6Rjc0aWtKZmYrZ2l0YlRaU1ZEL0RTWWp5UmJZa0sKTzJqMTJUTytJb0JObnNWRlAvVXRvY2JrNU1SWUxBb2c2aWJaMWJsTVdHSWl5TENaNGNlV1lUcGVJNVJTL2lzbgpQVllvck1rMzZicjN0WFdvcHJaVzZwT1NKeUNuTzA4VXZ3QmdIZ0VMOHJWYUVjdWhiUUlEQVFBQm8yRXdYekFPCkJnTlZIUThCQWY4RUJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEcKQTFVZEV3RUIvd1FGTUFNQkFmOHdIUVlEVlIwT0JCWUVGR0RLanJXNmJlT2RnS0QvYkFKUlRwTWhNc1hvTUEwRwpDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ1F3YUMycWVYSUxST1ZaTE9JTFAxNGh2WW55SElCMkdvQ2VpSHVvY0JOCll5dVhzamV4NWEzNDVYRGdGK1FsT0RpdnJ1WjlPMVdkUTk2UmZ2ZWQ3QnA3M2U3aHYrRVVmQUhJQlM4dXRIUzQKYVNBRjN0R3BoRTM2ck9KUzMwcmpBbkg5bzh0dWsxbVA1U2xOay9XUXlJdnc0cDROVFg2VzhRR3FQU01jL09INgpxVlF1YzlEdTlKcEFZYVVWOHBjZUhhK3BGVzlpUm9XL1Y5TDZINlBSREFyMWQ5OHA2MlBhdS9wbXV6Ni9kVW4zCmFvN0FnZHc3UTZRSnRsZ1g4NEdiblVadnZXVFdjZ05wZzZUcFJvdGx1SjBSbDlXZ1dDK0VxTjdQR3BmLzRWaTEKL3ZYRkQ0YWF0WWNTSUNieWtSejliRXlBeVNORHZxU0NTK0dPRmdrU29QWk4KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
   failurePolicy: Fail
   name: vmagent.victoriametrics.com
   admissionReviewVersions:
   - v1
   - v1beta1
   sideEffects: None
@@ -69,13 +69,13 @@

 - clientConfig:
     service:
       namespace: monitoring
       name: victoria-metrics-victoria-metrics-operator
       path: /validate-operator-victoriametrics-com-v1beta1-vmalert
       port: 9443
-    caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURIakNDQWdhZ0F3SUJBZ0lSQUsrY0d0ckhFNWtzKzA5ZGNjZ2hDaE13RFFZSktvWklodmNOQVFFTEJRQXcKR1RFWE1CVUdBMVVFQXhNT2RtMHRiM0JsY21GMGIzSXRZMkV3SGhjTk1qUXhNREV3TURreU5URTRXaGNOTXpReApNREE0TURreU5URTRXakFaTVJjd0ZRWURWUVFERXc1MmJTMXZjR1Z5WVhSdmNpMWpZVENDQVNJd0RRWUpLb1pJCmh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNUL2JXRStyREVnWUpiTGx2STUwQUE2YUUwK2JBVHBZWXoKOFRBK3lZenk2ZXhqUnVyUFVycXJqTWNGNEllbGpTbWNIMUZNcFM3c2s1Wlk0MmhhRnFCN29DM0RiUFBBRCtvTQpxcXNwR21tSGk5R1FVM3RxWWw5Tm9IcDkyTjJ2S2FXRXJVcHZacmk0V0lqUHpWQitHUGJPZzNLR2FHMVhjUkJVClI1a3gyeU1qMmh4MWtzUy96VERSamtldFBDU2daU05mWUI0dUc5SzhtVWZQa0ZLRTJVVnFCOTZVd3RMUWE4OEgKeFNQWnQxeFVHTFp1ay85bHRIbDFpZkJVL2pKY0VUNDdRRCtJR3VLUGpNM0hia1VqMHZOUU9RODdObWFiQ0t5UgpqMVNvQmhlN3ZSazRFM2lZTDBZaEhkLzZTZ3hjYkFKUGVYY05vU2N1NXpnWjBMTFZlRmtDQXdFQUFhTmhNRjh3CkRnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQVAKQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlROWG9mWCtqTkk1L1JGMlZ0dkJndkZmcFN0SURBTgpCZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFmQzYrcll3NFRKdW5KQklNTmFWRnZEdW1aUnc5UyttWE9lcU5wNEVPCnprK1k1V1F4K2luM0R6OTFBdENsMWRsRWpsdGQyeDJ4ajg4SjlqQ3JJaWVLeVJsVW5CNWdMVVp2RnFpbkx4VU0KelBZVTR0S3lXM2lDaUp0TnB4VGVRMXRVZUg4OWpYdDdrS05LYTUzb2NMWVRSWjJ3K2xXKzVnZVBZUGFhZkp4dApNMTgzRDFSWVM3TlZVZEhSYVVBRkRuSWRYOVJPMHQ3UUJSdm1KTlZSbE1GNENieW9iTzVXZUwvZ0ZMZGplaUNTCmYzZitTUXc5U1FEVFBVcllId2Z6ak90aUpWVGdaR1NiZjcvYXo5MUpvSElkSzVDdExDT2hxL1BqODNRSkxCRE4Kb1JoWjZFSzEzdVpyNFI3Yld2Kzk4MDBlYi9CUUpPc0J6RTd2ek9yeFBrVjIrUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-controller-manager

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-controller-manager

@@ -0,0 +1,19 @@

+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  namespace: monitoring
+  name: vmetrics-controller-manager
+  labels:
+    app: victoria-metrics-k8s-stack-grafana
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  allowCrossNamespaceImport: false
+  json: |
+    {"editable":false,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{ cluster=~\"$cluster\", job=\"kube-controller-manager\"})","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":20,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_adds_total{ cluster=~\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Add Rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_depth{ cluster=~\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Depth","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{ cluster=~\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work Queue Latency","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":21},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","legendFormat":"5xx"}],"title":"Kube API Request Rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":16,"x":8,"y":21},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ cluster=~\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))","legendFormat":"{{verb}} {{url}}"}],"title":"Post Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ cluster=~\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))","legendFormat":"{{verb}} {{url}}"}],"title":"Get Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":35},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{ cluster=~\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":35},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{ cluster=~\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU usage","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":35},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{ cluster=~\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin","vm-k8s-stack"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":".*","refresh":2,"sort":1,"type":"constant"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{ cluster=~\"$cluster\", job=\"kube-controller-manager\"}, instance)","refresh":2,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":"utc","title":"Kubernetes / Controller Manager","uid":"72e0e05bef5099e5f049b05fdc429ed4"}
+
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubelet

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubelet

@@ -0,0 +1,19 @@

+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  namespace: monitoring
+  name: vmetrics-kubelet
+  labels:
+    app: victoria-metrics-k8s-stack-grafana
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  allowCrossNamespaceImport: false
+  json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-system-api-server

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-system-api-server

@@ -0,0 +1,19 @@

+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  namespace: monitoring
+  name: vmetrics-kubernetes-system-api-server
+  labels:
+    app: victoria-metrics-k8s-stack-grafana
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  allowCrossNamespaceImport: false
+  json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-nodes

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-kubernetes-views-nodes

@@ -0,0 +1,19 @@

+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  namespace: monitoring
+  name: vmetrics-kubernetes-views-nodes
+  labels:
+    app: victoria-metrics-k8s-stack-grafana
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  allowCrossNamespaceImport: false
+  json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-scheduler

+++ HelmRelease: monitoring/victoria-metrics GrafanaDashboard: monitoring/vmetrics-scheduler

@@ -0,0 +1,19 @@

+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  namespace: monitoring
+  name: vmetrics-scheduler
+  labels:
+    app: victoria-metrics-k8s-stack-grafana
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana
+  allowCrossNamespaceImport: false
+  json: |
[Diff truncated by flux-local]
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containercpulimits

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containercpulimits

@@ -0,0 +1,38 @@

+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  namespace: monitoring
+  name: vmetrics-k8s.rules.containercpulimits
+  labels:
+    app: victoria-metrics-k8s-stack
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  groups:
+  - name: k8s.rules.container_cpu_limits
+    params: {}
+    rules:
+    - annotations: {}
+      expr: |-
+        kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}  * on (namespace,pod,cluster)
+        group_left() max by (namespace,pod,cluster) (
+         (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
+         )
+      labels: {}
+      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
+    - annotations: {}
+      expr: |-
+        sum by (namespace,cluster) (
+            sum by (namespace,pod,cluster) (
+                max by (namespace,pod,container,cluster) (
+                  kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace,pod,cluster) group_left() max by (namespace,pod,cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+            )
+        )
+      labels: {}
+      record: namespace_cpu:kube_pod_container_resource_limits:sum
+
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containercpurequests

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containercpurequests

@@ -0,0 +1,38 @@

+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  namespace: monitoring
+  name: vmetrics-k8s.rules.containercpurequests
+  labels:
+    app: victoria-metrics-k8s-stack
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  groups:
+  - name: k8s.rules.container_cpu_requests
+    params: {}
+    rules:
+    - annotations: {}
+      expr: |-
+        kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}  * on (namespace,pod,cluster)
+        group_left() max by (namespace,pod,cluster) (
+          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
+        )
+      labels: {}
+      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests
+    - annotations: {}
+      expr: |-
+        sum by (namespace,cluster) (
+            sum by (namespace,pod,cluster) (
+                max by (namespace,pod,container,cluster) (
+                  kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
+                ) * on (namespace,pod,cluster) group_left() max by (namespace,pod,cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+            )
+        )
+      labels: {}
+      record: namespace_cpu:kube_pod_container_resource_requests:sum
+
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containermemorylimits

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containermemorylimits

@@ -0,0 +1,38 @@

+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  namespace: monitoring
+  name: vmetrics-k8s.rules.containermemorylimits
+  labels:
+    app: victoria-metrics-k8s-stack
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  groups:
+  - name: k8s.rules.container_memory_limits
+    params: {}
+    rules:
+    - annotations: {}
+      expr: |-
+        kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}  * on (namespace,pod,cluster)
+        group_left() max by (namespace,pod,cluster) (
+          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
+        )
+      labels: {}
+      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits
+    - annotations: {}
+      expr: |-
+        sum by (namespace,cluster) (
+            sum by (namespace,pod,cluster) (
+                max by (namespace,pod,container,cluster) (
+                  kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace,pod,cluster) group_left() max by (namespace,pod,cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+            )
+        )
+      labels: {}
+      record: namespace_memory:kube_pod_container_resource_limits:sum
+
--- HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containermemoryrequests

+++ HelmRelease: monitoring/victoria-metrics VMRule: monitoring/vmetrics-k8s.rules.containermemoryrequests

@@ -0,0 +1,38 @@

+---
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  namespace: monitoring
+  name: vmetrics-k8s.rules.containermemoryrequests
+  labels:
+    app: victoria-metrics-k8s-stack
+    app.kubernetes.io/instance: victoria-metrics
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: victoria-metrics-k8s-stack
+spec:
+  groups:
+  - name: k8s.rules.container_memory_requests
+    params: {}
+    rules:
+    - annotations: {}
+      expr: |-
+        kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}  * on (namespace,pod,cluster)
+        group_left() max by (namespace,pod,cluster) (
+          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
+        )
+      labels: {}
+      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
+    - annotations: {}
+      expr: |-
+        sum by (namespace,cluster) (
+            sum by (namespace,pod,cluster) (
+                max by (namespace,pod,container,cluster) (
+                  kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
+                ) * on (namespace,pod,cluster) group_left() max by (namespace,pod,cluster) (
+                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                )
+            )
+        )
+      labels: {}
+      record: namespace_memory:kube_pod_container_resource_requests:sum
+