diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e6cfff..24fc4e08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,51 @@ # Change Log -## [2.19.2](https://github.com/bakdata/streams-bootstrap/tree/2.19.2) (2024-05-14) -[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.2...2.19.2) +## [2.23.0](https://github.com/bakdata/streams-bootstrap/tree/2.23.0) (2024-06-12) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.23.0...2.23.0) + + +## [2.23.0](https://github.com/bakdata/streams-bootstrap/tree/2.23.0) (2024-06-12) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.22.1...2.23.0) + +**Merged pull requests:** + +- Add dynamic application.server config to streams app chart [\#214](https://github.com/bakdata/streams-bootstrap/pull/214) ([@philipp94831](https://github.com/philipp94831)) + +## [2.22.1](https://github.com/bakdata/streams-bootstrap/tree/2.22.1) (2024-06-11) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.22.0...2.22.1) + +**Merged pull requests:** + +- Use same variable names for clean up job as for main chart [\#228](https://github.com/bakdata/streams-bootstrap/pull/228) ([@philipp94831](https://github.com/philipp94831)) + +## [2.22.0](https://github.com/bakdata/streams-bootstrap/tree/2.22.0) (2024-06-10) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.21.0...2.22.0) + +**Merged pull requests:** + +- Support mounting of external secrets [\#225](https://github.com/bakdata/streams-bootstrap/pull/225) ([@torbsto](https://github.com/torbsto)) + +## [2.21.0](https://github.com/bakdata/streams-bootstrap/tree/2.21.0) (2024-05-24) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.20.1...2.21.0) + +**Merged pull requests:** + +- Make affinity in charts more generic [\#224](https://github.com/bakdata/streams-bootstrap/pull/224) ([@yannick-roeder](https://github.com/yannick-roeder)) + +## [2.20.1](https://github.com/bakdata/streams-bootstrap/tree/2.20.1) (2024-05-23) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.20.0...2.20.1) + +**Merged pull requests:** + +- Roll deployments on config map updates [\#227](https://github.com/bakdata/streams-bootstrap/pull/227) ([@philipp94831](https://github.com/philipp94831)) +- Fix content of config maps [\#226](https://github.com/bakdata/streams-bootstrap/pull/226) ([@philipp94831](https://github.com/philipp94831)) + +## [2.20.0](https://github.com/bakdata/streams-bootstrap/tree/2.20.0) (2024-05-23) +[View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.2...2.20.0) + +**Merged pull requests:** +- Mount arbitrary files to apps [\#223](https://github.com/bakdata/streams-bootstrap/pull/223) ([@philipp94831](https://github.com/philipp94831)) ## [2.19.2](https://github.com/bakdata/streams-bootstrap/tree/2.19.2) (2024-05-14) [View commits](https://github.com/bakdata/streams-bootstrap/compare/2.19.1...2.19.2) diff --git a/charts/producer-app-cleanup-job/templates/_helpers.tpl b/charts/producer-app-cleanup-job/templates/_helpers.tpl index bd6f76e8..f21b03b8 100644 --- a/charts/producer-app-cleanup-job/templates/_helpers.tpl +++ b/charts/producer-app-cleanup-job/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "producer-app-cleanup-job.name" -}} +{{- define "producer-app.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -12,7 +12,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "producer-app-cleanup-job.fullname" -}} +{{- define "producer-app.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -24,6 +24,6 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "producer-app-cleanup-job.chart" -}} +{{- define "producer-app.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/charts/producer-app-cleanup-job/templates/configmap.yaml b/charts/producer-app-cleanup-job/templates/configmap.yaml index 8a0f6053..da5bb989 100644 --- a/charts/producer-app-cleanup-job/templates/configmap.yaml +++ b/charts/producer-app-cleanup-job/templates/configmap.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "producer-app-cleanup-job.name" . }} + name: {{ template "producer-app.name" . }} data: {{- range $key, $value := .Values.files }} - {{ $key }}: {{ $value.content | nindent 4 }} + {{ $key }}: {{ $value.content | quote }} {{- end }} {{ end }} diff --git a/charts/producer-app-cleanup-job/templates/job.yaml b/charts/producer-app-cleanup-job/templates/job.yaml index f747ed0b..65f11c2d 100644 --- a/charts/producer-app-cleanup-job/templates/job.yaml +++ b/charts/producer-app-cleanup-job/templates/job.yaml @@ -1,9 +1,8 @@ {{- $root := . -}} - apiVersion: batch/v1 kind: Job metadata: - name: {{ template "producer-app-cleanup-job.fullname" . }} + name: {{ template "producer-app.fullname" . }} {{- if .Values.annotations }} annotations: {{- range $key, $value := .Values.annotations }} @@ -11,8 +10,8 @@ metadata: {{- end }} {{- end }} labels: - app: {{ template "producer-app-cleanup-job.name" . }} - chart: {{ template "producer-app-cleanup-job.chart" . }} + app: {{ template "producer-app.name" . }} + chart: {{ template "producer-app.chart" . }} release: {{ .Release.Name }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value }} @@ -28,7 +27,7 @@ spec: {{- end }} {{- end }} labels: - app: {{ template "producer-app-cleanup-job.name" . }} + app: {{ template "producer-app.name" . }} release: {{ .Release.Name }} {{- range $key, $value := .Values.podLabels }} {{ $key }}: {{ $value }} @@ -42,17 +41,16 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8 }} {{- end }} + {{- with .Values.affinity }} affinity: - {{- if .Values.nodeAffinity }} - nodeAffinity: -{{ toYaml .Values.nodeAffinity | indent 10 }} - {{- end }} + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} containers: - - name: {{ template "producer-app-cleanup-job.name" . }} + - name: {{ template "producer-app.name" . }} image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" resources: @@ -90,7 +88,7 @@ spec: - name: "{{ $key }}" valueFrom: secretKeyRef: - name: {{ template "producer-app-cleanup-job.fullname" $ }} + name: {{ template "producer-app.fullname" $ }} key: "{{ $key }}" {{- end }} {{- range $key, $value := .Values.secretRefs }} @@ -111,18 +109,35 @@ spec: - name: JAVA_TOOL_OPTIONS value: '-XX:MaxRAMPercentage={{ printf "%.1f" .Values.javaOptions.maxRAMPercentage }} {{ .Values.javaOptions.others | join " " }}' - {{- if .Values.files }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumeMounts: {{- range $key, $value := .Values.files }} - name: config mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }} subPath: {{ $key | quote }} {{- end }} - {{- end }} - {{- if .Values.files }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + mountPath: {{ .mountPath }} + {{- if .readOnly }} + readOnly: true + {{- end }} + {{- if .subPath}} + subPath: {{.subPath }} + {{- end }} + {{- end }} + {{- end }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumes: + {{- if .Values.files }} - name: config configMap: - name: {{ template "producer-app-cleanup-job.name" . }} + name: {{ template "producer-app.name" . }} + {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + secret: + secretName: {{ .name }} + {{- end }} {{- end }} backoffLimit: {{ .Values.backoffLimit }} diff --git a/charts/producer-app-cleanup-job/templates/secrets.yaml b/charts/producer-app-cleanup-job/templates/secrets.yaml index 945f4a0a..7dccd797 100644 --- a/charts/producer-app-cleanup-job/templates/secrets.yaml +++ b/charts/producer-app-cleanup-job/templates/secrets.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ template "producer-app-cleanup-job.fullname" . }} + name: {{ template "producer-app.fullname" . }} type: Opaque data: {{- range $key, $value := .Values.secrets }} diff --git a/charts/producer-app-cleanup-job/values.yaml b/charts/producer-app-cleanup-job/values.yaml index d1cc1a2c..e0d956f7 100644 --- a/charts/producer-app-cleanup-job/values.yaml +++ b/charts/producer-app-cleanup-job/values.yaml @@ -40,6 +40,12 @@ secretRefs: {} # MY_SECRET: # name: secretName # key: secretKey +secretFilesRefs: [] +# - name: my-secret +# volume: secret-volume +# mountPath: /etc/test +# readOnly: true +# subPath: optional-subpath annotations: {} # MY_ANNOTATION: "foo-bar" @@ -57,24 +63,10 @@ tolerations: [] # operator: "Exists" # effect: "NoSchedule" -nodeAffinity: {} -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz -# preferredDuringSchedulingIgnoredDuringExecution: -# - weight: 1 -# preference: -# matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} resources: requests: diff --git a/charts/producer-app/README.md b/charts/producer-app/README.md index 67b57ac4..2366b8db 100644 --- a/charts/producer-app/README.md +++ b/charts/producer-app/README.md @@ -11,13 +11,13 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Job | Parameter | Description | Default | -|------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | | `nameOverride` | The name of the Kubernetes deployment. | `bakdata-producer-app` | | `resources` | See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | see [values.yaml](values.yaml) for details | | `annotations` | Map of custom annotations to attach to the deployment. | `{}` | | `labels` | Map of custom labels to attach to the deployment. | `{}` | | `tolerations` | Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `{}` | -| `nodeAffinity` | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity). | `{}` | +| `affinity` | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). | `{}` | | `deployment` | Deploy the producer as a Kubernetes Deployment (thereby ignoring Job-related configurations) | false | | `restartPolicy` | [Restart policy](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy) to use for the job. | `OnFailure` | | `schedule` | Cron expression to denote a schedule this producer app should be run on. It will then be deployed as a [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) instead of a [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/). | | @@ -57,13 +57,14 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Other | Parameter | Description | Default | -|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | | `configurationEnvPrefix` | Prefix for environment variables to use that should be parsed as command line arguments. | `APP` | | `commandLine` | Map of command line arguments passed to the producer app. | `{}` | | `debug` | Configure logging to debug | `false` | | `env` | Custom environment variables | `{}` | | `secrets` | Custom secret environment variables. Prefix with `configurationEnvPrefix` in order to pass secrets to command line or prefix with `STREAMS_` to pass secrets to Kafka Streams configuration. E.g., `APP_MY_PARAM` would be passed as `--my-param` and `STREAMS_MAX_POLL_TIMEOUT_MS` would be translated to `max.poll.timeout.ms`. | `{}` | | `secretRefs` | Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret `name` and `key`. | `{}` | +| `secretFilesRefs` | Mount existing secrets as volumes | `[]` | | `files` | Map of files to mount for the app. File will be mounted as `$value.mountPath/$key`. `$value.content` denotes file content (recommended to be used with `--set-file`). | `{}` | ### JVM diff --git a/charts/producer-app/templates/configmap.yaml b/charts/producer-app/templates/configmap.yaml index 04203ad1..da5bb989 100644 --- a/charts/producer-app/templates/configmap.yaml +++ b/charts/producer-app/templates/configmap.yaml @@ -5,6 +5,6 @@ metadata: name: {{ template "producer-app.name" . }} data: {{- range $key, $value := .Values.files }} - {{ $key }}: {{ $value.content | nindent 4 }} + {{ $key }}: {{ $value.content | quote }} {{- end }} {{ end }} diff --git a/charts/producer-app/templates/pod.yaml b/charts/producer-app/templates/pod.yaml index 96d60eae..772a716a 100644 --- a/charts/producer-app/templates/pod.yaml +++ b/charts/producer-app/templates/pod.yaml @@ -1,8 +1,11 @@ {{- define "producer-app.podTemplate" -}} {{- $root := . -}} metadata: -{{- if .Values.podAnnotations }} +{{- if or .Values.podAnnotations (and .Values.deployment .Values.files) }} annotations: + {{- if and .Values.deployment .Values.files }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} {{- range $key, $value := .Values.podAnnotations }} {{ $key | quote }}: {{ $value | quote }} {{- end }} @@ -24,11 +27,10 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 4 }} {{- end }} + {{- with .Values.affinity }} affinity: - {{- if .Values.nodeAffinity }} - nodeAffinity: -{{ toYaml .Values.nodeAffinity | indent 6 }} - {{- end }} + {{- tpl (toYaml .) $root | nindent 4 }} + {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 8 }} @@ -107,18 +109,35 @@ spec: readinessProbe: {{- .Values.readinessProbe | toYaml | nindent 6 }} {{- end }} - {{- if .Values.files }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumeMounts: {{- range $key, $value := .Values.files }} - name: config mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }} subPath: {{ $key | quote }} {{- end }} - {{- end }} - {{- if .Values.files }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + mountPath: {{ .mountPath }} + {{- if .readOnly }} + readOnly: true + {{- end }} + {{- if .subPath}} + subPath: {{.subPath }} + {{- end }} + {{- end }} + {{- end }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumes: + {{- if .Values.files }} - name: config configMap: - name: {{ template "producer-app.name" . }} + name: {{ template "producer-app.chart.name" . }} + {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + secret: + secretName: {{ .name }} + {{- end }} {{- end }} {{- end -}} diff --git a/charts/producer-app/values.yaml b/charts/producer-app/values.yaml index 708a7fd8..4f7480ab 100644 --- a/charts/producer-app/values.yaml +++ b/charts/producer-app/values.yaml @@ -35,24 +35,10 @@ tolerations: [] # operator: "Exists" # effect: "NoSchedule" -nodeAffinity: {} -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz -# preferredDuringSchedulingIgnoredDuringExecution: -# - weight: 1 -# preference: -# matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} resources: requests: @@ -87,6 +73,12 @@ secretRefs: {} # MY_SECRET: # name: secretName # key: secretKey +secretFilesRefs: [] +# - name: my-secret +# volume: secret-volume +# mountPath: /etc/test +# readOnly: true +# subPath: optional-subpath annotations: {} # MY_ANNOTATION: "foo-bar" diff --git a/charts/rclone-copy/templates/rclone-cron.yaml b/charts/rclone-copy/templates/rclone-cron.yaml index f644c017..7ddf601c 100644 --- a/charts/rclone-copy/templates/rclone-cron.yaml +++ b/charts/rclone-copy/templates/rclone-cron.yaml @@ -1,3 +1,4 @@ +{{- $root := . -}} {{- if .Capabilities.APIVersions.Has "batch/v1/CronJob" }} apiVersion: batch/v1 {{- else }} @@ -49,11 +50,10 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 12 }} {{- end }} + {{- with .Values.affinity }} affinity: - {{- if .Values.nodeAffinity }} - nodeAffinity: -{{ toYaml .Values.nodeAffinity | indent 14 }} - {{- end }} + {{- tpl (toYaml .) $root | nindent 12 }} + {{- end }} containers: - name: rclone-container image: rclone/rclone:{{ .Values.imageRelease }} diff --git a/charts/rclone-copy/values.yaml b/charts/rclone-copy/values.yaml index 5fc3632c..ace2fc06 100644 --- a/charts/rclone-copy/values.yaml +++ b/charts/rclone-copy/values.yaml @@ -56,24 +56,10 @@ tolerations: [] # operator: "Exists" # effect: "NoSchedule" -nodeAffinity: {} -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz -# preferredDuringSchedulingIgnoredDuringExecution: -# - weight: 1 -# preference: -# matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} resources: requests: diff --git a/charts/streams-app-cleanup-job/templates/_helpers.tpl b/charts/streams-app-cleanup-job/templates/_helpers.tpl index 3596a661..4944f922 100644 --- a/charts/streams-app-cleanup-job/templates/_helpers.tpl +++ b/charts/streams-app-cleanup-job/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "streams-app-cleanup-job.name" -}} +{{- define "streams-app.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -12,7 +12,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "streams-app-cleanup-job.fullname" -}} +{{- define "streams-app.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -24,6 +24,6 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "streams-app-cleanup-job.chart" -}} +{{- define "streams-app.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/charts/streams-app-cleanup-job/templates/configmap.yaml b/charts/streams-app-cleanup-job/templates/configmap.yaml index 6206009c..98c4aee3 100644 --- a/charts/streams-app-cleanup-job/templates/configmap.yaml +++ b/charts/streams-app-cleanup-job/templates/configmap.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ template "streams-app-cleanup-job.name" . }} + name: {{ template "streams-app.name" . }} data: {{- range $key, $value := .Values.files }} - {{ $key }}: {{ $value.content | nindent 4 }} + {{ $key }}: {{ $value.content | quote }} {{- end }} {{ end }} diff --git a/charts/streams-app-cleanup-job/templates/job.yaml b/charts/streams-app-cleanup-job/templates/job.yaml index 7d6020b9..7b19f207 100644 --- a/charts/streams-app-cleanup-job/templates/job.yaml +++ b/charts/streams-app-cleanup-job/templates/job.yaml @@ -1,9 +1,8 @@ {{- $root := . -}} - apiVersion: batch/v1 kind: Job metadata: - name: {{ template "streams-app-cleanup-job.fullname" . }} + name: {{ template "streams-app.fullname" . }} {{- if .Values.annotations }} annotations: {{- range $key, $value := .Values.annotations }} @@ -11,8 +10,8 @@ metadata: {{- end }} {{- end }} labels: - app: {{ template "streams-app-cleanup-job.name" . }} - chart: {{ template "streams-app-cleanup-job.chart" . }} + app: {{ template "streams-app.name" . }} + chart: {{ template "streams-app.chart" . }} release: {{ .Release.Name }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value }} @@ -28,7 +27,7 @@ spec: {{- end }} {{- end }} labels: - app: {{ template "streams-app-cleanup-job.name" . }} + app: {{ template "streams-app.name" . }} release: {{ .Release.Name }} {{- range $key, $value := .Values.podLabels }} {{ $key }}: {{ $value }} @@ -42,17 +41,16 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8 }} {{- end }} + {{- with .Values.affinity }} affinity: - {{- if .Values.nodeAffinity }} - nodeAffinity: -{{ toYaml .Values.nodeAffinity | indent 10 }} - {{- end }} + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} containers: - - name: {{ template "streams-app-cleanup-job.name" . }} + - name: {{ template "streams-app.name" . }} image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" resources: @@ -115,7 +113,7 @@ spec: - name: "{{ $key }}" valueFrom: secretKeyRef: - name: {{ template "streams-app-cleanup-job.fullname" $ }} + name: {{ template "streams-app.fullname" $ }} key: "{{ $key }}" {{- end }} {{- range $key, $value := .Values.secretRefs }} @@ -136,18 +134,35 @@ spec: - name: JAVA_TOOL_OPTIONS value: '-XX:MaxRAMPercentage={{ printf "%.1f" .Values.javaOptions.maxRAMPercentage }} {{ .Values.javaOptions.others | join " " }}' - {{- if .Values.files }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumeMounts: {{- range $key, $value := .Values.files }} - name: config mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }} subPath: {{ $key | quote }} {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + mountPath: {{ .mountPath }} + {{- if .readOnly }} + readOnly: true + {{- end }} + {{- if .subPath}} + subPath: {{.subPath }} + {{- end }} + {{- end }} {{- end }} - {{- if .Values.files }} + {{- if or (.Values.files) (.Values.secretFilesRefs) }} volumes: + {{- if .Values.files }} - name: config configMap: - name: {{ template "streams-app-cleanup-job.name" . }} + name: {{ template "streams-app.name" . }} + {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + secret: + secretName: {{ .name }} + {{- end }} {{- end }} backoffLimit: {{ .Values.backoffLimit }} diff --git a/charts/streams-app-cleanup-job/templates/secrets.yaml b/charts/streams-app-cleanup-job/templates/secrets.yaml index 20c099db..feb79e5b 100644 --- a/charts/streams-app-cleanup-job/templates/secrets.yaml +++ b/charts/streams-app-cleanup-job/templates/secrets.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ template "streams-app-cleanup-job.fullname" . }} + name: {{ template "streams-app.fullname" . }} type: Opaque data: {{- range $key, $value := .Values.secrets }} diff --git a/charts/streams-app-cleanup-job/values.yaml b/charts/streams-app-cleanup-job/values.yaml index df662764..b3464e2a 100644 --- a/charts/streams-app-cleanup-job/values.yaml +++ b/charts/streams-app-cleanup-job/values.yaml @@ -52,6 +52,12 @@ secretRefs: {} # MY_SECRET: # name: secretName # key: secretKey +secretFilesRefs: [] +# - name: my-secret +# volume: secret-volume +# mountPath: /etc/test +# readOnly: true +# subPath: optional-subpath annotations: {} # MY_ANNOTATION: "foo-bar" @@ -69,24 +75,11 @@ tolerations: [] # operator: "Exists" # effect: "NoSchedule" -nodeAffinity: {} -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz -# preferredDuringSchedulingIgnoredDuringExecution: -# - weight: 1 -# preference: -# matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} resources: requests: diff --git a/charts/streams-app/README.md b/charts/streams-app/README.md index 1a5c3dcd..77fa5803 100644 --- a/charts/streams-app/README.md +++ b/charts/streams-app/README.md @@ -11,7 +11,7 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Deployment | Parameter | Description | Default | -|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | | `nameOverride` | The name of the Kubernetes deployment. | `bakdata-streams-app` | | `replicaCount` | The number of Kafka Streams replicas. | `1` | | `resources` | See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | see [values.yaml](values.yaml) for details | @@ -20,8 +20,7 @@ Alternatively, a YAML file that specifies the values for the parameters can be p | `tolerations` | Array containing taint references. When defined, pods can run on nodes, which would otherwise deny scheduling. Further information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `{}` | | `statefulSet` | Whether to use a [Statefulset](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) instead of a [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) to deploy the streams app. | `false` | | `priorityClassName` | [Priority class name](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) for the pod. | | -| `nodeAffinity` | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity). | `{}` | -| `requirePodAntiAffinity` | Whether to require replicas to be scheduled on different nodes. See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity. | `false` | +| `affinity` | Map to configure [pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). | `{}` | | `ports.containerPort` | Number of the port to expose. | | | `ports.name` | Services can reference port by name (optional). | | | `ports.schema` | Protocol for port. Must be UDP, TCP, or SCTP (optional). | | @@ -69,13 +68,14 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Other | Parameter | Description | Default | -|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | | `configurationEnvPrefix` | Prefix for environment variables to use that should be parsed as command line arguments. | `APP` | | `commandLine` | Map of command line arguments passed to the streams app. | `{}` | | `debug` | Configure logging to debug | `false` | | `env` | Custom environment variables | `{}` | | `secrets` | Custom secret environment variables. Prefix with `configurationEnvPrefix` in order to pass secrets to command line or prefix with `STREAMS_` to pass secrets to Kafka Streams configuration. E.g., `APP_MY_PARAM` would be passed as `--my-param` and `STREAMS_MAX_POLL_TIMEOUT_MS` would be translated to `max.poll.timeout.ms`. | `{}` | | `secretRefs` | Inject existing secrets as environment variables. Map key is used as environment variable name. Value consists of secret `name` and `key`. | `{}` | +| `secretFilesRefs` | Mount existing secrets as volumes | `[]` | | `files` | Map of files to mount for the app. File will be mounted as `$value.mountPath/$key`. `$value.content` denotes file content (recommended to be used with `--set-file`). | `{}` | ### JMX Configuration @@ -99,7 +99,7 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Auto-Scaling | Parameter | Description | Default | -|----------------------------------|--------------------------------------------------------------------------------------------------------------------|------------| +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ---------- | | `autoscaling.enabled` | Whether to enable auto-scaling using [KEDA](https://keda.sh/docs/latest/scalers/apache-kafka/). | `false` | | `autoscaling.consumerGroup` | Name of the consumer group used for checking the offset on the topic and processing the related lag. | | | `autoscaling.lagThreshold` | Average target value to trigger scaling actions. | | @@ -122,8 +122,8 @@ Alternatively, a YAML file that specifies the values for the parameters can be p ### Service -| Parameter | Description | Default | -| ----------------- | ---------------------------------------------------------------------------------------------- | ----------- | -| `service.enabled` | Whether to create a service. This requires the definition of at least one `ports.servicePort`. | `false` | -| `service.labels` | Additional service labels. | `{}` | -| `service.type` | Service type. | `ClusterIP` | +| Parameter | Description | Default | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------| +| `service.enabled` | Whether to create a service. This requires the definition of at least one `ports.servicePort`. This also configures `application.server` for the streams app | `false` | +| `service.labels` | Additional service labels. | `{}` | +| `service.type` | Service type. | `ClusterIP` | diff --git a/charts/streams-app/templates/configmap.yaml b/charts/streams-app/templates/configmap.yaml index 8cb698e6..98c4aee3 100644 --- a/charts/streams-app/templates/configmap.yaml +++ b/charts/streams-app/templates/configmap.yaml @@ -5,6 +5,6 @@ metadata: name: {{ template "streams-app.name" . }} data: {{- range $key, $value := .Values.files }} - {{ $key }}: {{ $value.content | nindent 4 }} + {{ $key }}: {{ $value.content | quote }} {{- end }} {{ end }} diff --git a/charts/streams-app/templates/deployment.yaml b/charts/streams-app/templates/deployment.yaml index 2aac6915..c13b080c 100644 --- a/charts/streams-app/templates/deployment.yaml +++ b/charts/streams-app/templates/deployment.yaml @@ -39,8 +39,11 @@ spec: release: {{ .Release.Name }} template: metadata: - {{- if or .Values.prometheus.jmx.enabled .Values.podAnnotations }} + {{- if or .Values.prometheus.jmx.enabled .Values.podAnnotations .Values.files }} annotations: + {{- if .Values.files }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} {{- if .Values.prometheus.jmx.enabled }} prometheus.io/scrape: "true" prometheus.io/port: {{ .Values.prometheus.jmx.port | quote }} @@ -63,33 +66,10 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8 }} {{- end }} + {{- with .Values.affinity }} affinity: - {{- if .Values.nodeAffinity }} - nodeAffinity: -{{ toYaml .Values.nodeAffinity | indent 10 }} - {{- end }} - podAntiAffinity: - {{- if .Values.requirePodAntiAffinity }} - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ template "streams-app.name" . }} - topologyKey: "kubernetes.io/hostname" - {{- else }} - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - podAffinityTerm: - topologyKey: "kubernetes.io/hostname" - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - {{ template "streams-app.name" . }} - {{- end }} + {{- tpl (toYaml .) $root | nindent 8 }} + {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} @@ -111,6 +91,16 @@ spec: - name: {{ printf "STREAMS_%s" $key | replace "." "_" | upper | quote }} value: {{ $value | quote }} {{- end }} + {{- range .Values.ports }} + {{- if .servicePort }} # TODO verify that there is at most one service port. Currently, if there are multiple service ports, the first one will be used + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: STREAMS_APPLICATION_SERVER + value: "$(POD_IP):{{ .containerPort }}" + {{- end }} + {{- end }} {{- if .Values.streams.staticMembership }} - name: STREAMS_GROUP_INSTANCE_ID valueFrom: @@ -194,13 +184,23 @@ spec: -Dcom.sun.management.jmxremote.ssl=false -XX:MaxRAMPercentage={{ printf "%.1f" .Values.javaOptions.maxRAMPercentage }} {{ .Values.javaOptions.others | join " " }}' - {{- if or (.Values.files) (and .Values.persistence.enabled .Values.statefulSet) }} + {{- if or (.Values.files) (and .Values.persistence.enabled .Values.statefulSet) (.Values.secretFilesRefs) }} volumeMounts: {{- range $key, $value := .Values.files }} - name: config mountPath: {{ printf "%s/%s" $value.mountPath $key | quote }} subPath: {{ $key | quote }} {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + mountPath: {{ .mountPath }} + {{- if .readOnly }} + readOnly: true + {{- end }} + {{- if .subPath}} + subPath: {{.subPath }} + {{- end }} + {{- end }} {{- if and .Values.persistence.enabled .Values.statefulSet }} - name: datadir mountPath: /tmp/kafka-streams @@ -249,7 +249,7 @@ spec: - name: jmx-config mountPath: /etc/jmx-streams-app {{- end }} - {{- if or (.Values.prometheus.jmx.enabled) (.Values.files) }} + {{- if or (.Values.prometheus.jmx.enabled) (.Values.files) (.Values.secretFilesRefs) }} volumes: {{- if .Values.prometheus.jmx.enabled }} - name: jmx-config @@ -261,6 +261,11 @@ spec: configMap: name: {{ template "streams-app.name" . }} {{- end }} + {{- range .Values.secretFilesRefs }} + - name: {{ .volume }} + secret: + secretName: {{ .name }} + {{- end }} {{- end }} {{- if and .Values.persistence.enabled .Values.statefulSet }} volumeClaimTemplates: diff --git a/charts/streams-app/templates/service.yaml b/charts/streams-app/templates/service.yaml index 0a4588c8..df54dc0a 100644 --- a/charts/streams-app/templates/service.yaml +++ b/charts/streams-app/templates/service.yaml @@ -24,6 +24,6 @@ spec: protocol: {{ .protocol }} {{- end }} {{- end }} - {{- end }} + {{- end }} type: {{ .Values.service.type }} {{- end }} diff --git a/charts/streams-app/values.yaml b/charts/streams-app/values.yaml index f6d81bd7..b3627e9d 100644 --- a/charts/streams-app/values.yaml +++ b/charts/streams-app/values.yaml @@ -60,6 +60,12 @@ secretRefs: {} # MY_SECRET: # name: secretName # key: secretKey +secretFilesRefs: [] +# - name: my-secret +# volume: secret-volume +# mountPath: /etc/test +# readOnly: true +# subPath: optional-subpath # Additional ports ports: [] @@ -164,26 +170,10 @@ prometheus: # priorityClass # priorityClassName: "" -requirePodAntiAffinity: false - -nodeAffinity: {} -# requiredDuringSchedulingIgnoredDuringExecution: -# nodeSelectorTerms: -# - matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz -# preferredDuringSchedulingIgnoredDuringExecution: -# - weight: 1 -# preference: -# matchExpressions: -# - key: foo -# operator: In -# values: -# - bar -# - baz +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} persistence: enabled: false diff --git a/gradle.properties b/gradle.properties index 8b5862f1..6df93544 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=2.19.3-SNAPSHOT +version=2.23.1-SNAPSHOT org.gradle.caching=true # running Kafka JUnit in parallel causes problems org.gradle.parallel=false