feat: add servicemonitor for hydra & make servicemonitors configurable (

#427)
ory · Apr 8, 2022 · 72871d4 · 72871d4
1 parent 12dc589
commit 72871d4
Show file tree

Hide file tree

Showing 14 changed files with 191 additions and 8 deletions.
diff --git a/.circleci/values/hydra.yaml b/.circleci/values/hydra.yaml
@@ -41,4 +41,10 @@ job:
       image: "alpine:latest"
       command: ["/bin/sh"]
       args: ["-c", "sleep 10"]
-  shareProcessNamespace: true
+  shareProcessNamespace: true
+
+serviceMonitor:
+  labels:
+    release: "prometheus"
+  tlsConfig:
+    insecureSkipVerify: true
diff --git a/.circleci/values/keto.yaml b/.circleci/values/keto.yaml
@@ -24,4 +24,14 @@ job:
       image: "alpine:latest"
       command: ["/bin/sh"]
       args: ["-c", "sleep 10"]
-  shareProcessNamespace: true
+  shareProcessNamespace: true
+
+service:
+  metrics:
+    enabled: true
+
+serviceMonitor:
+  labels:
+    release: "prometheus"
+  tlsConfig: 
+    insecureSkipVerify: true
diff --git a/.circleci/values/kratos.yaml b/.circleci/values/kratos.yaml
@@ -101,4 +101,10 @@ job:
       image: "alpine:latest"
       command: ["/bin/sh"]
       args: ["-c", "sleep 10"]
-  shareProcessNamespace: true
+  shareProcessNamespace: true
+
+serviceMonitor:
+  labels:
+    release: "prometheus"
+  tlsConfig: 
+    insecureSkipVerify: true
diff --git a/.circleci/values/oathkeeper.yaml b/.circleci/values/oathkeeper.yaml
@@ -82,4 +82,16 @@ oathkeeper:
           "qi": "OdvyakUl-NZ2PZHi5N_vDTdC4Ad4LI6JP4InVW-33kGySQom264eZ-nwBFVlZeCx2qgFE0iuRtS1plmZdEMP_cc8kW-PDtTDg3i_8rWutkmX13FThHXpT9M3iTU8qxeizRuvXaHIayplgZT6W8iIl4JWp1lWfLK85jTmsuX2mF-I0E56VOGOy7xlBEnyrrskXgyiOcjFgNy2UTaCNvfUrLxhiWAU-ZoqyEaj4t5bYdcu_xkuwDvdHea9RgHOMve9UoSPsSIAoev1HeTdIrWLOyUEenGqhUAkneRqTDuXkzUYFreV63nhqiHU3WUGKBbJ-4Dgl7kl0FxH7w98WuKscg"
         }
       ]
-    }
+    }
+
+service:
+  metrics:
+    labels:
+      app.kubernetes.io/component: "metrics"
+      release: "metrics"
+
+serviceMonitor:
+  labels:
+    release: "prometheus"
+  tlsConfig: 
+    insecureSkipVerify: true
diff --git a/.circleci/values/prometheus.yaml b/.circleci/values/prometheus.yaml
@@ -0,0 +1,17 @@
+namespaceOverride: "prometheus"
+alertmanager:
+  enabled: false
+alertmanager:
+  enabled: false
+coreDns:
+  enabled: false
+kubeEtcd:
+  enabled: false
+kubeScheduler:
+  enabled: false
+kubeProxy:
+  enabled: false
+kubeStateMetrics:
+  enabled: false
+grafana:
+  enabled: true
diff --git a/Makefile b/Makefile
@@ -45,11 +45,17 @@ postgresql:
 	helm repo update
 	helm install postgresql bitnami/postgresql -f .circleci/values/postgres.yaml
 
+prometheus:
+	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+	helm repo update
+	kubectl create ns prometheus --dry-run=client -o yaml | kubectl apply -f -
+	helm install prometheus prometheus-community/kube-prometheus-stack -f .circleci/values/prometheus.yaml
+
 ory-repo:
 	helm repo add ory https://k8s.ory.sh/helm/charts
 	helm repo update
 
-kind-test: kind-start postgresql
+kind-test: kind-start postgresql prometheus
 	.circleci/helm-test.sh oathkeeper
 	.circleci/helm-test.sh oathkeeper-maester
 	.circleci/helm-test.sh hydra
@@ -58,7 +64,7 @@ kind-test: kind-start postgresql
 	.circleci/helm-test.sh keto
 	.circleci/helm-test.sh kratos-selfservice-ui-node
 
-kind-upgrade: kind-start postgresql ory-repo
+kind-upgrade: kind-start postgresql ory-repo prometheus
 	.circleci/helm-upgrade.sh oathkeeper
 	.circleci/helm-upgrade.sh oathkeeper-maester
 	.circleci/helm-upgrade.sh hydra

diff --git a/helm/charts/hydra/templates/service-admin.yaml b/helm/charts/hydra/templates/service-admin.yaml
@@ -1,4 +1,5 @@
 {{- if .Values.service.admin.enabled -}}
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,6 +10,7 @@ metadata:
     {{- with .Values.service.admin.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}
+    app.kubernetes.io/component: admin
   {{- with .Values.service.admin.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -23,4 +25,38 @@ spec:
   selector:
     app.kubernetes.io/name: {{ include "hydra.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") (.Values.serviceMonitor.enabled) }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "hydra.fullname" . }}-admin
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/component: admin
+{{ include "hydra.labels" . | indent 4 }}
+{{- with .Values.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+{{- end }}
+  {{- with .Values.service.admin.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - path: /metrics/prometheus
+    port: {{ .Values.service.admin.name }}
+    scheme: {{ .Values.serviceMonitor.scheme }}
+    interval: {{ .Values.serviceMonitor.scrapeInterval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    {{- with .Values.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "hydra.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: admin
+{{- end -}}
 {{- end }}
diff --git a/helm/charts/hydra/values.yaml b/helm/charts/hydra/values.yaml
@@ -376,3 +376,19 @@ pdb:
   enabled: false
   spec:
     minAvailable: 1
+
+# -- Parameters for the Prometheus ServiceMonitor objects. 
+# Reference: https://docs.openshift.com/container-platform/4.6/rest_api/monitoring_apis/servicemonitor-monitoring-coreos-com-v1.html
+serviceMonitor:
+  # -- switch to false to prevent creating the ServiceMonitor
+  enabled: true
+  # -- HTTP scheme to use for scraping.
+  scheme: https
+  # -- Interval at which metrics should be scraped
+  scrapeInterval: 60s
+  # -- Timeout after which the scrape is ended
+  scrapeTimeout: 30s
+  # -- Provide additionnal labels to the ServiceMonitor ressource metadata
+  labels: {}
+  # -- TLS configuration to use when scraping the endpoint
+  tlsConfig: {}
diff --git a/helm/charts/keto/templates/servicemonitor-metrics.yaml b/helm/charts/keto/templates/servicemonitor-metrics.yaml
@@ -8,6 +8,9 @@ metadata:
   labels:
     app.kubernetes.io/component: metrics
 {{ include "keto.labels" . | indent 4 }}
+  {{- with .Values.serviceMonitor.labels }}
+      {{- toYaml . | nindent 4 }}
+  {{- end }}
   {{- with .Values.service.metrics.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -16,6 +19,13 @@ spec:
   endpoints:
   - path: /metrics/prometheus
     port: {{ .Values.service.metrics.name }}
+    scheme: {{ .Values.serviceMonitor.scheme }}
+    interval: {{ .Values.serviceMonitor.scrapeInterval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    {{- with .Values.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "keto.name" . }}

diff --git a/helm/charts/keto/values.yaml b/helm/charts/keto/values.yaml
@@ -289,3 +289,17 @@ pdb:
   enabled: false
   spec:
     minAvailable: 1
+
+# -- Parameters for the Prometheus ServiceMonitor objects. 
+# Reference: https://docs.openshift.com/container-platform/4.6/rest_api/monitoring_apis/servicemonitor-monitoring-coreos-com-v1.html
+serviceMonitor:
+  # -- HTTP scheme to use for scraping.
+  scheme: https
+  # -- Interval at which metrics should be scraped
+  scrapeInterval: 60s
+  # -- Timeout after which the scrape is ended
+  scrapeTimeout: 30s
+  # -- Provide additionnal labels to the ServiceMonitor ressource metadata
+  labels: {}
+  # -- TLS configuration to use when scraping the endpoint
+  tlsConfig: {}
diff --git a/helm/charts/kratos/templates/service-admin.yaml b/helm/charts/kratos/templates/service-admin.yaml
@@ -25,7 +25,7 @@ spec:
   selector:
     app.kubernetes.io/name: {{ include "kratos.name" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
-{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" }}
+{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") (.Values.serviceMonitor.enabled) }}
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -35,6 +35,9 @@ metadata:
   labels:
     app.kubernetes.io/component: admin
 {{ include "kratos.labels" . | indent 4 }}
+{{- with .Values.serviceMonitor.labels }}
+  {{- toYaml . | nindent 4 }}
+{{- end }}
   {{- with .Values.service.admin.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -43,6 +46,13 @@ spec:
   endpoints:
   - path: /admin/metrics/prometheus
     port: {{ .Values.service.admin.name }}
+    scheme: {{ .Values.serviceMonitor.scheme }}
+    interval: {{ .Values.serviceMonitor.scrapeInterval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    {{- with .Values.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "kratos.name" . }}

diff --git a/helm/charts/kratos/values.yaml b/helm/charts/kratos/values.yaml
@@ -403,3 +403,20 @@ pdb:
   enabled: false
   spec:
     minAvailable: 1
+
+# -- Parameters for the Prometheus ServiceMonitor objects. 
+# Reference: https://docs.openshift.com/container-platform/4.6/rest_api/monitoring_apis/servicemonitor-monitoring-coreos-com-v1.html
+serviceMonitor:
+  # -- switch to false to prevent creating the ServiceMonitor
+  enabled: true
+  # -- HTTP scheme to use for scraping.
+  scheme: https
+  # -- Interval at which metrics should be scraped
+  scrapeInterval: 60s
+  # -- Timeout after which the scrape is ended
+  scrapeTimeout: 30s
+  # -- Provide additionnal labels to the ServiceMonitor ressource metadata
+  labels: {}
+  # -- TLS configuration to use when scraping the endpoint
+  tlsConfig: {}
+
diff --git a/helm/charts/oathkeeper/templates/service-metrics.yaml b/helm/charts/oathkeeper/templates/service-metrics.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.service.metrics.enabled }}
 ---
 apiVersion: v1
 kind: Service
@@ -36,7 +37,7 @@ metadata:
   labels:
     app.kubernetes.io/component: metrics
 {{ include "oathkeeper.labels" . | indent 4 }}
-    {{- with .Values.service.metrics.labels }}
+    {{- with merge .Values.serviceMonitor.labels .Values.service.metrics.labels }}
       {{- toYaml . | nindent 4 }}
     {{- end }}
   {{- with .Values.service.metrics.annotations }}
@@ -47,9 +48,17 @@ spec:
   endpoints:
   - path: /metrics/prometheus
     port: {{ .Values.service.metrics.name }}
+    scheme: {{ .Values.serviceMonitor.scheme }}
+    interval: {{ .Values.serviceMonitor.scrapeInterval }}
+    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    {{- with .Values.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ include "oathkeeper.name" . }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: metrics
+{{- end -}}
 {{- end -}}
diff --git a/helm/charts/oathkeeper/values.yaml b/helm/charts/oathkeeper/values.yaml
@@ -296,3 +296,17 @@ pdb:
   enabled: false
   spec:
     minAvailable: 1
+
+# -- Parameters for the Prometheus ServiceMonitor objects. 
+# Reference: https://docs.openshift.com/container-platform/4.6/rest_api/monitoring_apis/servicemonitor-monitoring-coreos-com-v1.html
+serviceMonitor:
+  # -- HTTP scheme to use for scraping.
+  scheme: https
+  # -- Interval at which metrics should be scraped
+  scrapeInterval: 60s
+  # -- Timeout after which the scrape is ended
+  scrapeTimeout: 30s
+  # -- Provide additionnal labels to the ServiceMonitor ressource metadata
+  labels: {}
+  # -- TLS configuration to use when scraping the endpoint
+  tlsConfig: {}