feat: add watcher pilot (#293)

This PR introduces a sidecar container to each deployment using the DSN mechanism. It allows setting a watcher for a mounted file, and restarting the deployment upon changes in the file. This uses inotify-tools, which report upon file changes. In this case we are operating with k8s volumes, which are mounted in some location and symlinked into the desired location. Change to the original file (so unmount and remount) causes the symlink to be deleted and recreated. In some cases this can break the program (f.e if the mounted file is a secret/certificate used by DSN) requiring a restart.
ory · Jul 9, 2021 · 1103402 · 1103402
1 parent 0fc5506
commit 1103402
Show file tree

Hide file tree

Showing 14 changed files with 280 additions and 20 deletions.
diff --git a/.circleci/values/hydra.yaml b/.circleci/values/hydra.yaml
@@ -6,4 +6,18 @@ hydra:
     secrets:
       system: OG5XbmxXa3dYeGplQXpQanYxeEFuRUFa
 maester:
-  enabled: false
+  enabled: false
+
+# deployment:
+#  extraVolumes:
+#   - name: my-volume
+#     secret:
+#       secretName: my-secret
+#   extraVolumeMounts:
+#   - name: my-volume
+#     mountPath: /etc/secrets/my-secret
+#     readOnly: true
+
+# watcher:
+#   enabled: true
+#   mountFile: /etc/secrets/my-secret/foo
diff --git a/helm/charts/hydra/files/watch.sh b/helm/charts/hydra/files/watch.sh
@@ -0,0 +1,13 @@
+set -eu
+
+function rollOut() {
+  DEPLOY=$(kubectl get deploy -n "${NAMESPACE}" -l "${1}" -o name)
+  kubectl set env -n $NAMESPACE ${DEPLOY} sync=$(date "+%Y%m%d-%H%M%S")
+  kubectl rollout status -n $NAMESPACE ${DEPLOY}
+}
+
+inotifywait -e DELETE_SELF -m "${WATCH_FILE}" |
+   while read path _ file; do
+       echo "---> $path$file modified"
+       rollOut "${DEPLOYMENT_SELECTOR}"
+   done
diff --git a/helm/charts/hydra/templates/deployment.yaml b/helm/charts/hydra/templates/deployment.yaml
@@ -43,11 +43,31 @@ spec:
         {{- if .Values.deployment.extraVolumes }}
 {{ toYaml .Values.deployment.extraVolumes | indent 8 }}
         {{- end }}
-      {{- if .Values.deployment.serviceAccountName  }}
-      serviceAccountName: {{ .Values.deployment.serviceAccountName }}
-      {{- end }}
+      serviceAccountName: {{ .Values.deployment.serviceAccountName  | default ( include "hydra.fullname" . ) }}
       automountServiceAccountToken: {{ .Values.deployment.automountServiceAccountToken }}
       containers:
+        {{- if .Values.watcher.enabled }}
+        - name: watcher
+          securityContext:
+            {{- toYaml .Values.deployment.securityContext | nindent 12 }}
+          image: {{ .Values.watcher.image }}
+          command:
+            - /bin/bash
+            - -c
+            - |
+{{ .Files.Get "files/watch.sh" | printf "%s" | indent 14 }}
+          env:
+            - name: NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+            - name: WATCH_FILE
+              value: {{ .Values.watcher.mountFile | quote }}
+            - name: DEPLOYMENT_SELECTOR
+              value: 'app.kubernetes.io/name={{ include "hydra.name" . }}'
+          volumeMounts:
+          {{- with .Values.deployment.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}

diff --git a/helm/charts/hydra/templates/rbac.yaml b/helm/charts/hydra/templates/rbac.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "hydra.fullname" . }}
+  labels:
+    {{- include "hydra.labels" . | nindent 4 }}
+
+{{- if .Values.watcher.enabled }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "hydra.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "patch", "get", "watch"]
+    resourceNames:
+      - {{ include "hydra.fullname" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "hydra.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "hydra.fullname" . }}-watcher
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "hydra.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/charts/hydra/values.yaml b/helm/charts/hydra/values.yaml
@@ -210,7 +210,13 @@ deployment:
   # serviceAccountName:
 
   extraVolumes: []
+  # - name: my-volume
+  #   secret:
+  #     secretName: my-secret
   extraVolumeMounts: []
+  # - name: my-volume
+  #   mountPath: /etc/secrets/my-secret
+  #   readOnly: true
   # If you want to mount external volume
   # For example, mount a secret containing Certificate root CA to verify database
   # TLS connection.
@@ -242,7 +248,7 @@ deployment:
     failureThreshold: 5
 
   # https://github.com/kubernetes/kubernetes/issues/57601
-  automountServiceAccountToken: false
+  automountServiceAccountToken: true
 
 job:
   annotations: {}
@@ -265,3 +271,9 @@ hydra-maester:
     # You only need to set this port if you change the value for
     # `service.admin.port` in the parent chart
     # port:
+
+watcher:
+  enabled: false
+  image: oryd/k8s-toolbox:0.0.1
+  mountFile: ''
+  # mountFile: /etc/secrets/my-secret/foo
diff --git a/helm/charts/keto/files/watch.sh b/helm/charts/keto/files/watch.sh
@@ -0,0 +1,13 @@
+set -eu
+
+function rollOut() {
+  DEPLOY=$(kubectl get deploy -n "${NAMESPACE}" -l "${1}" -o name)
+  kubectl set env -n $NAMESPACE ${DEPLOY} sync=$(date "+%Y%m%d-%H%M%S")
+  kubectl rollout status -n $NAMESPACE ${DEPLOY}
+}
+
+inotifywait -e DELETE_SELF -m "${WATCH_FILE}" |
+   while read path _ file; do
+       echo "---> $path$file modified"
+       rollOut "${DEPLOYMENT_SELECTOR}"
+   done
diff --git a/helm/charts/keto/templates/deployment.yaml b/helm/charts/keto/templates/deployment.yaml
@@ -29,6 +29,28 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       containers:
+        {{- if .Values.watcher.enabled }}
+        - name: watcher
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: {{ .Values.watcher.image }}
+          command:
+            - /bin/bash
+            - -c
+            - |
+{{ .Files.Get "files/watch.sh" | printf "%s" | indent 14 }}
+          env:
+            - name: NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+            - name: WATCH_FILE
+              value: {{ .Values.watcher.mountFile | quote }}
+            - name: DEPLOYMENT_SELECTOR
+              value: 'app.kubernetes.io/name={{ include "keto.name" . }}'
+          volumeMounts:
+          {{- with .Values.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}

diff --git a/helm/charts/keto/templates/rbac.yaml b/helm/charts/keto/templates/rbac.yaml
@@ -0,0 +1,46 @@
+{{- if .Values.serviceAccount.create -}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "keto.serviceAccountName" . }}
+  labels:
+    {{- include "keto.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+
+{{- if .Values.watcher.enabled }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "keto.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "patch", "get", "watch"]
+    resourceNames:
+      - {{ include "keto.fullname" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "keto.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "keto.fullname" . }}-watcher
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "keto.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
+
+{{- end }}
diff --git a/helm/charts/keto/templates/serviceaccount.yaml b/helm/charts/keto/templates/serviceaccount.yaml
diff --git a/helm/charts/keto/values.yaml b/helm/charts/keto/values.yaml
@@ -29,7 +29,7 @@ podSecurityContext: {}
   # fsGroup: 2000
 
 # https://github.com/kubernetes/kubernetes/issues/57601
-automountServiceAccountToken: false
+automountServiceAccountToken: true
 
 securityContext:
   capabilities:
@@ -177,4 +177,10 @@ deployment:
   readinessProbe:
     initialDelaySeconds: 30
     periodSeconds: 10
-    failureThreshold: 5
+    failureThreshold: 5
+
+watcher:
+  enabled: false
+  image: oryd/k8s-toolbox:0.0.1
+  mountFile: ''
+  # mountFile: /etc/secrets/my-secret/foo
diff --git a/helm/charts/kratos/files/watch.sh b/helm/charts/kratos/files/watch.sh
@@ -0,0 +1,13 @@
+set -eu
+
+function rollOut() {
+  DEPLOY=$(kubectl get deploy -n "${NAMESPACE}" -l "${1}" -o name)
+  kubectl set env -n $NAMESPACE ${DEPLOY} sync=$(date "+%Y%m%d-%H%M%S")
+  kubectl rollout status -n $NAMESPACE ${DEPLOY}
+}
+
+inotifywait -e DELETE_SELF -m "${WATCH_FILE}" |
+   while read path _ file; do
+       echo "---> $path$file modified"
+       rollOut "${DEPLOYMENT_SELECTOR}"
+   done
diff --git a/helm/charts/kratos/templates/deployment.yaml b/helm/charts/kratos/templates/deployment.yaml
@@ -61,7 +61,30 @@ spec:
         {{- end }}
         {{- end }}
       automountServiceAccountToken: {{ .Values.automountServiceAccountToken }}
+      serviceAccountName: {{ include "kratos.fullname" . }}
       containers:
+        {{- if .Values.watcher.enabled }}
+        - name: watcher
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: {{ .Values.watcher.image }}
+          command:
+            - /bin/bash
+            - -c
+            - |
+{{ .Files.Get "files/watch.sh" | printf "%s" | indent 14 }}
+          env:
+            - name: NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+            - name: WATCH_FILE
+              value: {{ .Values.watcher.mountFile | quote }}
+            - name: DEPLOYMENT_SELECTOR
+              value: 'app.kubernetes.io/name={{ include "kratos.name" . }}'
+          volumeMounts:
+          {{- with .Values.deployment.extraVolumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}        
         - name: {{ .Chart.Name }}
           image: {{ include "kratos.image" . }}
           imagePullPolicy: {{ include "kratos.imagePullPolicy" . }}

diff --git a/helm/charts/kratos/templates/rbac.yaml b/helm/charts/kratos/templates/rbac.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "kratos.fullname" . }}
+  labels:
+    {{- include "kratos.labels" . | nindent 4 }}
+
+{{- if .Values.watcher.enabled }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "kratos.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "patch", "get", "watch"]
+    resourceNames:
+      - {{ include "kratos.fullname" . }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "kratos.fullname" . }}-watcher
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "kratos.fullname" . }}-watcher
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "kratos.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/charts/kratos/values.yaml b/helm/charts/kratos/values.yaml
@@ -165,7 +165,13 @@ deployment:
 
   extraEnv: []
   extraVolumes: []
+  # - name: my-volume
+  #   secret:
+  #     secretName: my-secret
   extraVolumeMounts: []
+  # - name: my-volume
+  #   mountPath: /etc/secrets/my-secret
+  #   readOnly: true
   # If you want to mount external volume
   # For example, mount a secret containing Certificate root CA to verify database
   # TLS connection.
@@ -247,7 +253,7 @@ deployment:
   # environmentSecretsName:
 
   # https://github.com/kubernetes/kubernetes/issues/57601
-  automountServiceAccountToken: false
+  automountServiceAccountToken: true
 
 securityContext:
   capabilities:
@@ -278,3 +284,9 @@ nodeSelector: {}
 #   foo: bar
 # Configure node tolerations.
 tolerations: []
+
+watcher:
+  enabled: false
+  image: oryd/k8s-toolbox:0.0.1
+  mountFile: ''
+  # mountFile: /etc/secrets/my-secret/foo