feat(): support alertmanager basic auth (update opensource and ruler …

…chart to 1.0.4) (#20) ## What type of PR is this? * [x] Feature * [ ] BUG * [ ] Alerts * [x] Improvement * [ ] Documentation * [x] Test and CI ## Which issue(s) this PR related: issue # matrixorigin/MO-Cloud#3817 ## What this PR does / why we need it: 1. alertmanager 添加 basic auth鉴权 2. alertmanager 支持 ha 集群部署 3. 添加基于 tilt 的本地集成测试环境部署脚本
matrixorigin · Aug 23, 2024 · 4a56589 · 4a56589
1 parent 28a45d2
commit 4a56589
Show file tree

Hide file tree

Showing 18 changed files with 737 additions and 126 deletions.
diff --git a/README.md b/README.md
@@ -36,7 +36,6 @@ GRAFANA_USER=<your-admin-user>
 GRAFANA_PWD=<your-grafana-pwd>
 MO_RULER_STACK_VERSION=<helm version>
 MO_OB_OPENSOURCE_VERSION=<helm version>
-CONTROLPLANE_RESOURCE_CHART_VERSION=<helm version>
 ```
 
 ### 部署 mo-ruler-stack
@@ -136,6 +135,67 @@ kubectl get secret -n ${OBNS} grafana-admin-secret  -o jsonpath="{.data['admin-u
 kubectl get secret -n ${OBNS} grafana-admin-secret  -o jsonpath="{.data['admin-password']}" | base64 -d
 ```
 
+
+# 进阶配置
+
+
+## alertmanger 打开 web 鉴权
+
+1.在 `charts/mo-ruler-stack/values.yaml` 下设置 secretValue.alertmanager，alertmanager_web_auth_password_bcrypted 是 alertmanager_web_auth_password 的 bcrypt 加密
+
+```
+# secret value to create secret automatically
+secretValue:
+  alertmanager: 
+    # see: https://prometheus.io/docs/alerting/0.25/https
+    alertmanager_web_auth_user: admin
+    alertmanager_web_auth_password: admin
+    # need to be bcrypted, in bash: htpasswd -bnBC 10 "" <alertmanager_web_auth_password> | tr -d ':\n'
+    alertmanager_web_auth_password_bcrypted: $2y$10$Z3zgfm2IIeQqNmGWeqsrSecRuRmo/EAh4Srn0Mi0fG98dJZMn7RTS
+```
+
+2.在 `charts/mo-ruler-stack/values.yaml` 下启用 web.config.file:
+```
+alertmanager:
+  extraArgs:
+    web.config.file: /tmp/alertmanager-web-config/alertmanager-web-config.yaml
+```
+
+
+## 开启 alertmanager 鉴权与 alertmanager ha集群模式
+
+需要修改以下配置：
+
+1.在 `charts/mo-ruler-stack/values.yaml` 下修改 replicaCount:
+```
+alertmanager:
+  replicaCount: 3
+```
+
+
+2.在 `charts/mo-ob-opensource/values.yaml` 下修改 prometheus 的 alertingEndpoints 启用多个 alertmanager
+
+```
+kube-prometheus-stack:
+  prometheus:
+    pometheusSpec:
+      alertingEndpoints:
+      - name: "mo-ob-alertmanager-0"
+      - name: "mo-ob-alertmanager-1"
+      - name: "mo-ob-alertmanager-2"
+```
+
+3.在 `charts/mo-ob-opensource/values.yaml` 下修改 loki 的 alertmanager_url 启用多个 alertmanager
+
+```
+loki:
+  loki:
+    rulerConfig:
+      alertmanager_url: http://mo-ob-alertmanager-0.mo-ob:9093,http://mo-ob-alertmanager-1.mo-ob:9093,http://mo-ob-alertmanager-2.mo-ob:9093
+```
+
+即可启用 alertmanager ha集群
+
 # Scrape
 
 [Scrape List](./docs/scrape/README.md) 

diff --git a/Tiltfile b/Tiltfile
@@ -0,0 +1,45 @@
+load('ext://helm_remote', 'helm_remote')
+helm_remote(
+  'operator', 
+  repo_url='https://operator.min.io', 
+  release_name='minio-operator', 
+  namespace='minio-operator', 
+  version='6.0.2',
+  create_namespace=True,
+)
+
+helm_remote(
+  'tenant', 
+  repo_url='https://operator.min.io', 
+  release_name='loki-tenant', 
+  namespace='loki-tenant', 
+  version='6.0.2',    
+  values=['./dev/loki-tenant.yaml'],
+  create_namespace=True,
+)
+
+# 设置 Helm Chart 的本地路径
+mo_ob_opensource_chart = './charts/mo-ob-opensource'
+mo_ruler_stack_chart = './charts/mo-ruler-stack'
+
+local('kubectl get ns mo-ob || kubectl create ns mo-ob')
+
+k8s_yaml(
+  helm(
+    mo_ruler_stack_chart,
+    name='mo-ruler-stack',
+    namespace='mo-ob',
+    values=['./dev/mo-ruler-stack.dev.yaml'],
+  )
+)
+
+k8s_yaml(
+  helm(
+    mo_ob_opensource_chart,
+    name='mo-ob-opensource',
+    namespace='mo-ob',
+    values=['./dev/mo-ob-opensource.dev.yaml'],
+  )
+)
+
+k8s_yaml("./dev/loki_test_rule.yaml")
diff --git a/charts/mo-ob-opensource/Chart.yaml b/charts/mo-ob-opensource/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: mo-ob-opensource
 description: mo-ob-opensource's Helm chart for Kubernetes
 type: application
-version: 1.0.3-alpha.2
+version: 1.0.4
 appVersion: 0.9.0
 dependencies:
 - condition: kube-prometheus-stack.enabled

diff --git a/charts/mo-ob-opensource/values.yaml b/charts/mo-ob-opensource/values.yaml
@@ -156,7 +156,17 @@ loki:
         type: local
         local: 
           directory: /rules
+      # alertmanager 单节点
       alertmanager_url: http://mo-ob-alertmanager.mo-ob:9093
+      # alertmanager 3节点集群使用：
+      # alertmanager_url: http://mo-ob-alertmanager-0.mo-ob:9093,http://mo-ob-alertmanager-1.mo-ob:9093,http://mo-ob-alertmanager-2.mo-ob:9093 
+      enable_alertmanager_v2: true
+      enable_api: true
+      alertmanager_client:
+        type: "Basic"
+        credentials_file: "/tmp/loki/alertmanager-loki-credentials"
+
+
     # -- Additional query scheduler config
     ### use default config, old config not work in loki 3.0.0
     #storage_config:
@@ -233,6 +243,15 @@ loki:
       limits:
         memory: "2Gi"
         cpu: "2000m"
+    # -- Volume mounts to add to the backend pods
+    extraVolumeMounts:
+    - name: alertmanager-credentials
+      mountPath: /tmp/loki
+      readOnly: true
+    extraVolumes: 
+    - name: alertmanager-credentials
+      secret:
+        secretName: alertmanager-loki-credentials
 
   # Configuration for the gateway
   gateway:
@@ -570,13 +589,65 @@ kube-prometheus-stack:
               target_label: pod
 
       alertingEndpoints:
+      # 单节点部署
         - name: "mo-ob-alertmanager"
           # 如果跨ns，需要修改
           namespace: "mo-ob"
           port: 9093
           scheme: http
           pathPrefix: ""
           apiVersion: v2
+          basicAuth:
+            username: 
+              key: username
+              name: alertmanager-auth-secret
+            password: 
+              key: password
+              name: alertmanager-auth-secret
+      #集群部署
+        # - name: "mo-ob-alertmanager-0"
+        #   # 如果跨ns，需要修改
+        #   namespace: "mo-ob"
+        #   port: 9093
+        #   scheme: http
+        #   pathPrefix: ""
+        #   apiVersion: v2
+        #   basicAuth:
+        #     username: 
+        #       key: username
+        #       name: alertmanager-auth-secret
+        #     password: 
+        #       key: password
+        #       name: alertmanager-auth-secret
+        # - name: "mo-ob-alertmanager-1"
+        #   # 如果跨ns，需要修改
+        #   namespace: "mo-ob"
+        #   port: 9093
+        #   scheme: http
+        #   pathPrefix: ""
+        #   apiVersion: v2
+        #   basicAuth:
+        #     username: 
+        #       key: username
+        #       name: alertmanager-auth-secret
+        #     password: 
+        #       key: password
+        #       name: alertmanager-auth-secret
+        # - name: "mo-ob-alertmanager-2"
+        #   # 如果跨ns，需要修改
+        #   namespace: "mo-ob"
+        #   port: 9093
+        #   scheme: http
+        #   pathPrefix: ""
+        #   apiVersion: v2
+        #   basicAuth:
+        #     username: 
+        #       key: username
+        #       name: alertmanager-auth-secret
+        #     password: 
+        #       key: password
+        #       name: alertmanager-auth-secret
+
 
   thanosRuler:
     enabled: false
diff --git a/charts/mo-ruler-stack/Chart.yaml b/charts/mo-ruler-stack/Chart.yaml
@@ -2,13 +2,13 @@ apiVersion: v2
 name: mo-ruler-stack
 description: mo-ruler's Helm chart for Kubernetes
 type: application
-version: 1.0.3-alpha.1
+version: 1.0.4
 appVersion: 0.9.0
 dependencies:
 - condition: alertmanager.enabled
   name: alertmanager
   repository: https://prometheus-community.github.io/helm-charts
-  version: 1.2.0
+  version: 1.12.0
 - condition: grafana.enabled
   name: grafana
   repository: https://grafana.github.io/helm-charts

diff --git a/charts/mo-ruler-stack/templates/_helpers.tpl b/charts/mo-ruler-stack/templates/_helpers.tpl
@@ -277,3 +277,12 @@ global:
   {{- end }}
 {{- end }}
 {{- end -}}
+
+{{ define "alertmanager.web" }}
+basic_auth_users:
+  {{ .Values.secretValue.alertmanager.alertmanager_web_auth_user }}: {{ .Values.secretValue.alertmanager.alertmanager_web_auth_password_bcrypted }}
+{{ end }}
+
+{{- define "alertmanager.web-digest" -}}
+{{ include "alertmanager.web" . | sha256sum | trunc 8 }}
+{{- end -}}
diff --git a/charts/mo-ruler-stack/templates/alertmanager-auth-secret.yaml b/charts/mo-ruler-stack/templates/alertmanager-auth-secret.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-auth-secret
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+data:
+  username: {{ .Values.secretValue.alertmanager.alertmanager_web_auth_user | b64enc }}
+  password: {{ .Values.secretValue.alertmanager.alertmanager_web_auth_password | b64enc }}
+type: Opaque
diff --git a/charts/mo-ruler-stack/templates/alertmanager-datasource.yaml b/charts/mo-ruler-stack/templates/alertmanager-datasource.yaml
@@ -22,8 +22,8 @@ data:
         # Whether or not Grafana should send alert instances to this Alertmanager
         handleGrafanaManagedAlerts: false
       # optionally
-      # basicAuth: true
-      # basicAuthUser: my_user
-      # secureJsonData:
-      #   basicAuthPassword: test_password
+      basicAuth: true
+      basicAuthUser: {{ .Values.secretValue.alertmanager.alertmanager_web_auth_user }}
+      secureJsonData:
+        basicAuthPassword: {{ .Values.secretValue.alertmanager.alertmanager_web_auth_password }} 
 {{- end -}}
diff --git a/charts/mo-ruler-stack/templates/alertmanager-loki-credentials.yaml b/charts/mo-ruler-stack/templates/alertmanager-loki-credentials.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-loki-credentials
+data:
+  alertmanager-loki-credentials: {{ printf "%s:%s" .Values.secretValue.alertmanager.alertmanager_web_auth_user .Values.secretValue.alertmanager.alertmanager_web_auth_password | b64enc | b64enc }}
+type: Opaque
diff --git a/charts/mo-ruler-stack/templates/alertmanager-web-config.yaml b/charts/mo-ruler-stack/templates/alertmanager-web-config.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-web-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ .Release.Name }}
+data:
+  alertmanager-web-config.yaml: {{ include "alertmanager.web" . | b64enc }}