Merge remote-tracking branch 'origin/master' into feature/topology-bu…

…ilder
bakdata · Jun 12, 2024 · 66f74e9 · 66f74e9
2 parents 71d325d + c3c3fa2
commit 66f74e9
Show file tree

Hide file tree

Showing 23 changed files with 245 additions and 194 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,8 +1,51 @@
 # Change Log
 
-## [2.19.2](https://github.com/bakdata/streams-bootstrap/tree/2.19.2) (2024-05-14)
-[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.2...2.19.2)
+## [2.23.0](https://github.com/bakdata/streams-bootstrap/tree/2.23.0) (2024-06-12)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.23.0...2.23.0)
+
+
+## [2.23.0](https://github.com/bakdata/streams-bootstrap/tree/2.23.0) (2024-06-12)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.22.1...2.23.0)
+
+**Merged pull requests:**
+
+- Add dynamic application.server config to streams app chart [\#214](https://github.com/bakdata/streams-bootstrap/pull/214) ([@philipp94831](https://github.com/philipp94831))
+
+## [2.22.1](https://github.com/bakdata/streams-bootstrap/tree/2.22.1) (2024-06-11)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.22.0...2.22.1)
+
+**Merged pull requests:**
+
+- Use same variable names for clean up job as for main chart [\#228](https://github.com/bakdata/streams-bootstrap/pull/228) ([@philipp94831](https://github.com/philipp94831))
+
+## [2.22.0](https://github.com/bakdata/streams-bootstrap/tree/2.22.0) (2024-06-10)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.21.0...2.22.0)
+
+**Merged pull requests:**
+
+- Support mounting of external secrets [\#225](https://github.com/bakdata/streams-bootstrap/pull/225) ([@torbsto](https://github.com/torbsto))
+
+## [2.21.0](https://github.com/bakdata/streams-bootstrap/tree/2.21.0) (2024-05-24)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.20.1...2.21.0)
+
+**Merged pull requests:**
+
+- Make affinity in charts more generic [\#224](https://github.com/bakdata/streams-bootstrap/pull/224) ([@yannick-roeder](https://github.com/yannick-roeder))
+
+## [2.20.1](https://github.com/bakdata/streams-bootstrap/tree/2.20.1) (2024-05-23)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.20.0...2.20.1)
+
+**Merged pull requests:**
+
+- Roll deployments on config map updates [\#227](https://github.com/bakdata/streams-bootstrap/pull/227) ([@philipp94831](https://github.com/philipp94831))
+- Fix content of config maps [\#226](https://github.com/bakdata/streams-bootstrap/pull/226) ([@philipp94831](https://github.com/philipp94831))
+
+## [2.20.0](https://github.com/bakdata/streams-bootstrap/tree/2.20.0) (2024-05-23)
+[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.2...2.20.0)
+
+**Merged pull requests:**
 
+- Mount arbitrary files to apps [\#223](https://github.com/bakdata/streams-bootstrap/pull/223) ([@philipp94831](https://github.com/philipp94831))
 
 ## [2.19.2](https://github.com/bakdata/streams-bootstrap/tree/2.19.2) (2024-05-14)
 [View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.1...2.19.2)

diff --git a/charts/producer-app-cleanup-job/templates/_helpers.tpl b/charts/producer-app-cleanup-job/templates/_helpers.tpl
@@ -2,7 +2,7 @@
 {{/*
 Expand the name of the chart.
 */}}
-{{- define "producer-app-cleanup-job.name" -}}
+{{- define "producer-app.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
@@ -12,7 +12,7 @@ Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
-{{- define "producer-app-cleanup-job.fullname" -}}
+{{- define "producer-app.fullname" -}}
 {{- if .Values.fullnameOverride -}}
 {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
@@ -24,6 +24,6 @@ If release name contains chart name it will be used as a full name.
 {{/*
 Create chart name and version as used by the chart label.
 */}}
-{{- define "producer-app-cleanup-job.chart" -}}
+{{- define "producer-app.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
diff --git a/charts/producer-app-cleanup-job/templates/configmap.yaml b/charts/producer-app-cleanup-job/templates/configmap.yaml
@@ -2,9 +2,9 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ template "producer-app-cleanup-job.name" . }}
+  name: {{ template "producer-app.name" . }}
 data:
   {{- range $key, $value := .Values.files }}
-  {{ $key }}: {{ $value.content | nindent 4 }}
+  {{ $key }}: {{ $value.content | quote }}
   {{- end }}
 {{ end }}
diff --git a/charts/producer-app-cleanup-job/templates/job.yaml b/charts/producer-app-cleanup-job/templates/job.yaml
@@ -1,18 +1,17 @@
 {{- $root := . -}}
-
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ template "producer-app-cleanup-job.fullname" . }}
+  name: {{ template "producer-app.fullname" . }}
 {{- if .Values.annotations }}
   annotations:
   {{- range $key, $value := .Values.annotations }}
     {{ $key | quote }}: {{ $value | quote }}
   {{- end }}
 {{- end }}
   labels:
-    app: {{ template "producer-app-cleanup-job.name" . }}
-    chart: {{ template "producer-app-cleanup-job.chart" . }}
+    app: {{ template "producer-app.name" . }}
+    chart: {{ template "producer-app.chart" . }}
     release: {{ .Release.Name }}
     {{- range $key, $value := .Values.labels }}
     {{ $key }}: {{ $value }}
@@ -28,7 +27,7 @@ spec:
       {{- end }}
     {{- end }}
       labels:
-        app: {{ template "producer-app-cleanup-job.name" . }}
+        app: {{ template "producer-app.name" . }}
         release: {{ .Release.Name }}
         {{- range $key, $value := .Values.podLabels }}
         {{ $key }}: {{ $value }}
@@ -42,17 +41,16 @@ spec:
       tolerations:
 {{ toYaml .Values.tolerations | indent 8 }}
       {{- end }}
+      {{- with .Values.affinity }}
       affinity:
-        {{- if .Values.nodeAffinity }}
-        nodeAffinity:
-{{ toYaml .Values.nodeAffinity | indent 10 }}
-        {{- end }}
+        {{- tpl (toYaml .) $root | nindent 8 }}
+      {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
 {{- toYaml .Values.imagePullSecrets | nindent 8 }}
       {{- end }}
       containers:
-        - name: {{ template "producer-app-cleanup-job.name" . }}
+        - name: {{ template "producer-app.name" . }}
           image: "{{ .Values.image }}:{{ .Values.imageTag }}"
           imagePullPolicy: "{{ .Values.imagePullPolicy }}"
           resources:
@@ -90,7 +88,7 @@ spec:
             - name: "{{ $key }}"
               valueFrom:
                 secretKeyRef:
-                  name: {{ template "producer-app-cleanup-job.fullname" $ }}
+                  name: {{ template "producer-app.fullname" $ }}
                   key: "{{ $key }}"
             {{- end }}
             {{- range $key, $value := .Values.secretRefs }}
@@ -111,18 +109,35 @@ spec:
             - name: JAVA_TOOL_OPTIONS
               value: '-XX:MaxRAMPercentage={{ printf "%.1f" .Values.javaOptions.maxRAMPercentage }}
                 {{ .Values.javaOptions.others | join " " }}'
-        {{- if .Values.files }}
+          {{- if or (.Values.files) (.Values.secretFilesRefs) }}
           volumeMounts:
             {{- range $key, $value := .Values.files }}
             - name: config
               mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }}
               subPath: {{ $key | quote }}
             {{- end }}
-        {{- end }}
-      {{- if .Values.files }}
+            {{- range .Values.secretFilesRefs }}
+            - name: {{ .volume }}
+              mountPath: {{ .mountPath }}
+              {{- if .readOnly }}
+              readOnly: true
+              {{- end }}
+              {{- if .subPath}}
+              subPath: {{.subPath }}
+              {{- end }}
+            {{- end }}
+          {{- end }}
+      {{- if or (.Values.files) (.Values.secretFilesRefs) }}
       volumes:
+        {{- if .Values.files }}
         - name: config
           configMap:
-            name: {{ template "producer-app-cleanup-job.name" . }}
+            name: {{ template "producer-app.name" . }}
+        {{- end }}
+        {{- range .Values.secretFilesRefs }}
+        - name: {{ .volume }}
+          secret:
+            secretName: {{ .name }}
+        {{- end }}
       {{- end }}
   backoffLimit: {{ .Values.backoffLimit }}
diff --git a/charts/producer-app-cleanup-job/templates/secrets.yaml b/charts/producer-app-cleanup-job/templates/secrets.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: {{ template "producer-app-cleanup-job.fullname" . }}
+  name: {{ template "producer-app.fullname" . }}
 type: Opaque
 data:
   {{- range $key, $value := .Values.secrets }}

diff --git a/charts/producer-app-cleanup-job/values.yaml b/charts/producer-app-cleanup-job/values.yaml
@@ -40,6 +40,12 @@ secretRefs: {}
 #  MY_SECRET:
 #    name: secretName
 #    key: secretKey
+secretFilesRefs: []
+#  - name: my-secret
+#    volume: secret-volume
+#    mountPath: /etc/test
+#    readOnly: true
+#    subPath: optional-subpath
 
 annotations: {}
 #  MY_ANNOTATION: "foo-bar"
@@ -57,24 +63,10 @@ tolerations: []
 #     operator: "Exists"
 #     effect: "NoSchedule"
 
-nodeAffinity: {}
-#   requiredDuringSchedulingIgnoredDuringExecution:
-#     nodeSelectorTerms:
-#     - matchExpressions:
-#       - key: foo
-#         operator: In
-#         values:
-#         - bar
-#         - baz
-#   preferredDuringSchedulingIgnoredDuringExecution:
-#   - weight: 1
-#     preference:
-#       matchExpressions:
-#       - key: foo
-#         operator: In
-#         values:
-#         - bar
-#         - baz
+## Affinity for pod assignment (evaluated as template)
+## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+##
+affinity: {}
 
 resources:
   requests:

diff --git a/charts/producer-app/README.md b/charts/producer-app/README.md
@@ -11,13 +11,13 @@ Alternatively, a YAML file that specifies the values for the parameters can be p
 ### Job
 
 | Parameter                    | Description                                                                                                                                                                                                                                                                | Default                                    |
-|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|
+| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ |
 | `nameOverride`               | The name of the Kubernetes deployment.                                                                                                                                                                                                                                     | `bakdata-producer-app`                     |
 | `resources`                  | See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/                                                                                                                                                                                         | see [values.yaml](values.yaml) for details |
 | `annotations`                | Map of custom annotations to attach to the deployment.                                                                                                                                                                                                                     | `{}`                                       |
 | `labels`                     | Map of custom labels to attach to the deployment.                                                                                                                                                                                                                          | `{}`                                       |
 | `tolerations`                | Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)                 | `{}`                                       |
-| `nodeAffinity`               | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity).                                                                                                                                                 | `{}`                                       |
+| `affinity`                   | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity).                                                                                                                                                 | `{}`                                       |
 | `deployment`                 | Deploy the producer as a Kubernetes Deployment (thereby ignoring Job-related configurations)                                                                                                                                                                               | false                                      |
 | `restartPolicy`              | [Restart policy](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy) to use for the job.                                                                                                                                                     | `OnFailure`                                |
 | `schedule`                   | Cron expression to denote a schedule this producer app should be run on. It will then be deployed as a [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) instead of a [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/). |                                            |
@@ -57,13 +57,14 @@ Alternatively, a YAML file that specifies the values for the parameters can be p
 ### Other
 
 | Parameter                | Description                                                                                                                                                                                                                                                                                                                       | Default |
-|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|
+| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
 | `configurationEnvPrefix` | Prefix for environment variables to use that should be parsed as command line arguments.                                                                                                                                                                                                                                          | `APP`   |
 | `commandLine`            | Map of command line arguments passed to the producer app.                                                                                                                                                                                                                                                                         | `{}`    |
 | `debug`                  | Configure logging to debug                                                                                                                                                                                                                                                                                                        | `false` |
 | `env`                    | Custom environment variables                                                                                                                                                                                                                                                                                                      | `{}`    |
 | `secrets`                | Custom secret environment variables. Prefix with `configurationEnvPrefix` in order to pass secrets to command line or prefix with `STREAMS_` to pass secrets to Kafka Streams configuration. E.g., `APP_MY_PARAM` would be passed as `--my-param` and `STREAMS_MAX_POLL_TIMEOUT_MS` would be translated to `max.poll.timeout.ms`. | `{}`    |
 | `secretRefs`             | Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret `name` and `key`.                                                                                                                                                                                        | `{}`    |
+| `secretFilesRefs`        | Mount existing secrets as volumes                                                                                                                                                                                                                                                                                                 | `[]`    |
 | `files`                  | Map of files to mount for the app. File will be mounted as `$value.mountPath/$key`. `$value.content` denotes file content (recommended to be used with `--set-file`).                                                                                                                                                             | `{}`    |
 
 ### JVM

diff --git a/charts/producer-app/templates/configmap.yaml b/charts/producer-app/templates/configmap.yaml
@@ -5,6 +5,6 @@ metadata:
   name: {{ template "producer-app.name" . }}
 data:
   {{- range $key, $value := .Values.files }}
-  {{ $key }}: {{ $value.content | nindent 4 }}
+  {{ $key }}: {{ $value.content | quote }}
   {{- end }}
 {{ end }}
diff --git a/charts/producer-app/templates/pod.yaml b/charts/producer-app/templates/pod.yaml
@@ -1,8 +1,11 @@
 {{- define "producer-app.podTemplate" -}}
 {{- $root := . -}}
 metadata:
-{{- if .Values.podAnnotations }}
+{{- if or .Values.podAnnotations (and .Values.deployment .Values.files) }}
   annotations:
+  {{- if and .Values.deployment .Values.files }}
+    checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+  {{- end }}
   {{- range $key, $value := .Values.podAnnotations }}
     {{ $key | quote }}: {{ $value | quote }}
   {{- end }}
@@ -24,11 +27,10 @@ spec:
   tolerations:
 {{ toYaml .Values.tolerations | indent 4 }}
   {{- end }}
+  {{- with .Values.affinity }}
   affinity:
-    {{- if .Values.nodeAffinity }}
-    nodeAffinity:
-{{ toYaml .Values.nodeAffinity | indent 6 }}
-    {{- end }}
+    {{- tpl (toYaml .) $root | nindent 4 }}
+  {{- end }}
   {{- if .Values.imagePullSecrets }}
   imagePullSecrets:
 {{- toYaml .Values.imagePullSecrets | nindent 8 }}
@@ -107,18 +109,35 @@ spec:
     readinessProbe:
     {{- .Values.readinessProbe | toYaml | nindent 6 }}
     {{- end }}
-  {{- if .Values.files }}
+    {{- if or (.Values.files) (.Values.secretFilesRefs) }}
     volumeMounts:
       {{- range $key, $value := .Values.files }}
       - name: config
         mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }}
         subPath: {{ $key | quote }}
       {{- end }}
-  {{- end }}
-  {{- if .Values.files }}
+      {{- range .Values.secretFilesRefs }}
+      - name: {{ .volume }}
+        mountPath: {{ .mountPath }}
+        {{- if .readOnly }}
+        readOnly: true
+        {{- end }}
+        {{- if .subPath}}
+        subPath: {{.subPath }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+  {{- if or (.Values.files) (.Values.secretFilesRefs) }}
   volumes:
+    {{- if .Values.files }}
     - name: config
       configMap:
-        name: {{ template "producer-app.name" . }}
+        name: {{ template "producer-app.chart.name" . }}
+    {{- end }}
+    {{- range .Values.secretFilesRefs }}
+    - name: {{ .volume }}
+      secret:
+        secretName: {{ .name }}
+    {{- end }}
   {{- end }}
 {{- end -}}