From 8e52252bcd8333b23b150ee952e842e981639d03 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Thu, 17 Apr 2025 13:57:11 +0200 Subject: [PATCH 1/4] Refactor PostgreSQL configuration and remove deprecated database setup - Introduced a unified PostgreSQL configuration structure in values.yaml, replacing the old db configuration. - Added new helper functions for managing PostgreSQL environment variables and secrets based on the selected configuration type (postgrescluster, external-plaintext, external-secret). - Removed old database-related templates (ConfigMap, Deployment, PVC, Secrets, Service) that are no longer needed. - Updated the pgstacbootstrap job and configmap templates to align with the new PostgreSQL configuration. - Implemented validation for PostgreSQL settings to ensure required fields are set based on the selected type. --- helm-chart/eoapi/templates/_helpers.tpl | 251 +++++++++++++++--- helm-chart/eoapi/templates/db/configmap.yaml | 38 --- helm-chart/eoapi/templates/db/deployment.yaml | 76 ------ helm-chart/eoapi/templates/db/pvc.yaml | 16 -- helm-chart/eoapi/templates/db/secrets.yaml | 20 -- helm-chart/eoapi/templates/db/service.yaml | 16 -- .../configmap.yaml | 2 +- .../job.yaml | 42 ++- .../eoapi/templates/services/deployment.yaml | 16 +- helm-chart/eoapi/values.yaml | 69 ++--- 10 files changed, 269 insertions(+), 277 deletions(-) delete mode 100644 helm-chart/eoapi/templates/db/configmap.yaml delete mode 100644 helm-chart/eoapi/templates/db/deployment.yaml delete mode 100644 helm-chart/eoapi/templates/db/pvc.yaml delete mode 100644 helm-chart/eoapi/templates/db/secrets.yaml delete mode 100644 helm-chart/eoapi/templates/db/service.yaml rename helm-chart/eoapi/templates/{pgstacboostrap => pgstacbootstrap}/configmap.yaml (95%) rename helm-chart/eoapi/templates/{pgstacboostrap => pgstacbootstrap}/job.yaml (76%) diff --git a/helm-chart/eoapi/templates/_helpers.tpl b/helm-chart/eoapi/templates/_helpers.tpl index d419ec61..6abc43a7 100644 --- a/helm-chart/eoapi/templates/_helpers.tpl +++ b/helm-chart/eoapi/templates/_helpers.tpl @@ -62,63 +62,82 @@ Create the name of the service account to use {{- end }} {{/* -Secrets for postgres/postgis access have to be -derived from what the crunchydata operator creates +PostgreSQL environment variables based on the configured type +*/}} +{{- define "eoapi.postgresqlEnv" -}} +{{- if eq .Values.postgresql.type "postgrescluster" }} + {{- include "eoapi.postgresclusterSecrets" . }} +{{- else if eq .Values.postgresql.type "external-plaintext" }} + {{- include "eoapi.externalPlaintextPgSecrets" . }} +{{- else if eq .Values.postgresql.type "external-secret" }} + {{- include "eoapi.externalSecretPgSecrets" . }} +{{- end }} +{{- end }} -Also note that we want to use the pgbouncer- -but currently it doesn't support `search_path` parameters -(https://github.com/pgbouncer/pgbouncer/pull/73) which -are required for much of *pgstac +{{/* +PostgreSQL cluster secrets */}} -{{- define "eoapi.pgstacSecrets" -}} +{{- define "eoapi.postgresclusterSecrets" -}} {{- range $userName, $v := .Values.postgrescluster.users -}} {{/* do not render anything for the "postgres" user */}} {{- if not (eq (index $v "name") "postgres") }} -- name: POSTGRES_USER +# Standard PostgreSQL environment variables +- name: PGUSER valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} key: user -- name: POSTGRES_PORT +- name: PGPORT valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} key: port -- name: POSTGRES_HOST +- name: PGHOST valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} key: host -- name: POSTGRES_HOST_READER +- name: PGPASSWORD valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: host -- name: POSTGRES_HOST_WRITER + key: password +- name: PGDATABASE valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: host -- name: POSTGRES_PASS + key: dbname +- name: PGBOUNCER_URI valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: password -- name: POSTGRES_DBNAME + key: pgbouncer-uri +# Legacy variables for backward compatibility +- name: POSTGRES_USER valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: dbname -- name: PGBOUNCER_URI + key: user +- name: POSTGRES_PORT valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: pgbouncer-uri -- name: DATABASE_URL + key: port +- name: POSTGRES_HOST valueFrom: secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} - key: uri + key: host +- name: POSTGRES_PASS + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: password +- name: POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: dbname {{- end }} {{- end }} - name: PGADMIN_URI @@ -128,6 +147,180 @@ are required for much of *pgstac key: uri {{- end }} +{{/* +External PostgreSQL with plaintext credentials +*/}} +{{- define "eoapi.externalPlaintextPgSecrets" -}} +# Standard PostgreSQL environment variables +- name: PGUSER + value: {{ .Values.postgresql.external.credentials.username | quote }} +- name: PGPORT + value: {{ .Values.postgresql.external.port | quote }} +- name: PGHOST + value: {{ .Values.postgresql.external.host | quote }} +- name: PGPASSWORD + value: {{ .Values.postgresql.external.credentials.password | quote }} +- name: PGDATABASE + value: {{ .Values.postgresql.external.database | quote }} +# Legacy variables for backward compatibility +- name: POSTGRES_USER + value: {{ .Values.postgresql.external.credentials.username | quote }} +- name: POSTGRES_PORT + value: {{ .Values.postgresql.external.port | quote }} +- name: POSTGRES_HOST + value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_PASS + value: {{ .Values.postgresql.external.credentials.password | quote }} +- name: POSTGRES_DBNAME + value: {{ .Values.postgresql.external.database | quote }} +{{- end }} + +{{/* +External PostgreSQL with secret credentials +*/}} +{{- define "eoapi.externalSecretPgSecrets" -}} +# Standard PostgreSQL environment variables +- name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.username }} +- name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.password }} +# Legacy variables for backward compatibility +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.username }} +- name: POSTGRES_PASS + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.password }} + +# Host, port, and database can be from the secret or from values +{{- if .Values.postgresql.external.existingSecret.keys.host }} +- name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.host }} +- name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.host }} +{{- else }} +- name: PGHOST + value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_HOST + value: {{ .Values.postgresql.external.host | quote }} +{{- end }} + +{{- if .Values.postgresql.external.existingSecret.keys.port }} +- name: PGPORT + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.port }} +- name: POSTGRES_PORT + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.port }} +{{- else }} +- name: PGPORT + value: {{ .Values.postgresql.external.port | quote }} +- name: POSTGRES_PORT + value: {{ .Values.postgresql.external.port | quote }} +{{- end }} + +{{- if .Values.postgresql.external.existingSecret.keys.database }} +- name: PGDATABASE + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.database }} +- name: POSTGRES_DBNAME + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.database }} +{{- else }} +- name: PGDATABASE + value: {{ .Values.postgresql.external.database | quote }} +- name: POSTGRES_DBNAME + value: {{ .Values.postgresql.external.database | quote }} +{{- end }} +{{- end }} + +{{/* +Validate PostgreSQL configuration +*/}} +{{- define "eoapi.validatePostgresql" -}} +{{- if eq .Values.postgresql.type "postgrescluster" }} + {{- if not .Values.postgrescluster.enabled }} + {{- fail "When postgresql.type is 'postgrescluster', postgrescluster.enabled must be true" }} + {{- end }} + {{- include "eoapi.validatePostgresCluster" . }} +{{- else if eq .Values.postgresql.type "external-plaintext" }} + {{- if not .Values.postgresql.external.host }} + {{- fail "When postgresql.type is 'external-plaintext', postgresql.external.host must be set" }} + {{- end }} + {{- if not .Values.postgresql.external.credentials.username }} + {{- fail "When postgresql.type is 'external-plaintext', postgresql.external.credentials.username must be set" }} + {{- end }} + {{- if not .Values.postgresql.external.credentials.password }} + {{- fail "When postgresql.type is 'external-plaintext', postgresql.external.credentials.password must be set" }} + {{- end }} +{{- else if eq .Values.postgresql.type "external-secret" }} + {{- if not .Values.postgresql.external.existingSecret.name }} + {{- fail "When postgresql.type is 'external-secret', postgresql.external.existingSecret.name must be set" }} + {{- end }} + {{- if not .Values.postgresql.external.existingSecret.keys.username }} + {{- fail "When postgresql.type is 'external-secret', postgresql.external.existingSecret.keys.username must be set" }} + {{- end }} + {{- if not .Values.postgresql.external.existingSecret.keys.password }} + {{- fail "When postgresql.type is 'external-secret', postgresql.external.existingSecret.keys.password must be set" }} + {{- end }} + {{- if not .Values.postgresql.external.existingSecret.keys.host }} + {{- if not .Values.postgresql.external.host }} + {{- fail "When postgresql.type is 'external-secret' and existingSecret.keys.host is not set, postgresql.external.host must be set" }} + {{- end }} + {{- end }} +{{- else }} + {{- fail "postgresql.type must be one of: 'postgrescluster', 'external-plaintext', 'external-secret'" }} +{{- end }} +{{- end }} + +{{/* +Map legacy configuration to new postgresql configuration +*/}} +{{- define "eoapi.mapLegacyPostgresql" -}} +{{- $postgresql := dict }} +{{- if .Values.postgrescluster.enabled }} + {{- $_ := set $postgresql "type" "postgrescluster" }} +{{- else if .Values.db.enabled }} + {{- $_ := set $postgresql "type" "external-plaintext" }} + {{- $external := dict }} + {{- $_ := set $external "host" .Values.db.settings.secrets.POSTGRES_HOST }} + {{- $_ := set $external "port" .Values.db.settings.secrets.POSTGRES_PORT }} + {{- $_ := set $external "database" .Values.db.settings.secrets.POSTGRES_DB }} + {{- $credentials := dict }} + {{- $_ := set $credentials "username" .Values.db.settings.secrets.POSTGRES_USER }} + {{- $_ := set $credentials "password" .Values.db.settings.secrets.POSTGRES_PASSWORD }} + {{- $_ := set $external "credentials" $credentials }} + {{- $_ := set $postgresql "external" $external }} +{{- else }} + {{- $_ := set $postgresql "type" "postgrescluster" }} +{{- end }} +{{- $postgresql | toYaml }} +{{- end }} + {{/* values.schema.json doesn't play nice combined value checks so we use this helper function to check autoscaling rules @@ -192,17 +385,3 @@ that you can only use traefik as ingress when `testing=true` {{- end -}} {{- end -}} - -{{/* -validate: -that you cannot have db.enabled and (postgrescluster.enabled or pgstacBootstrap.enabled) -*/}} -{{- define "eoapi.validateTempDB" -}} -{{- if and (.Values.db.enabled) (.Values.postgrescluster.enabled) -}} - {{- fail "you cannot use have both db.enabled and postgresclsuter.enabled" -}} -{{- end -}} -{{- if and (.Values.db.enabled) (.Values.pgstacBootstrap.enabled) -}} - {{- fail "you cannot use have both db.enabled and pgstacBootstrap.enabled" -}} -{{- end -}} - -{{- end -}} diff --git a/helm-chart/eoapi/templates/db/configmap.yaml b/helm-chart/eoapi/templates/db/configmap.yaml deleted file mode 100644 index 00d5d5b9..00000000 --- a/helm-chart/eoapi/templates/db/configmap.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{- if .Values.db.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-sql-config-{{ $.Release.Name }} -data: - initdb.sql: | - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.sql" -}} - {{ $.Files.Get $path | nindent 4 }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-json-config-{{ $.Release.Name }} -data: - {{- range $path, $bytes := $.Files.Glob "initdb-data/*.json" -}} - {{- base $path | nindent 2 -}}: | {{- $.Files.Get $path | nindent 4 -}} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: initdb-sh-config-{{ $.Release.Name }} -data: - load.sh: | - #!/bin/bash - apt update -y && apt install python3 python3-pip -y - pip install pypgstac[psycopg]=={{ .Values.db.image.tag | trimPrefix "v" }} - DSN="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" - pypgstac pgready --dsn $DSN - pypgstac load collections /opt/initdb/json-data/noaa-emergency-response.json --dsn $DSN --method insert_ignore - pypgstac load items /opt/initdb/json-data/noaa-eri-nashville2020.json --dsn $DSN --method insert_ignore - psql $DSN -f /opt/initdb/sql-data/initdb.sql - echo "DONE LOADING!!!!!!" - # run it forever like a docker process should - tail -f /dev/null -{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/deployment.yaml b/helm-chart/eoapi/templates/db/deployment.yaml deleted file mode 100644 index 12f3e708..00000000 --- a/helm-chart/eoapi/templates/db/deployment.yaml +++ /dev/null @@ -1,76 +0,0 @@ -{{- if .Values.db.enabled }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pgstac - labels: - app: pgstac -spec: - selector: - matchLabels: - app: pgstac - strategy: - type: Recreate - template: - metadata: - labels: - app: pgstac - spec: - restartPolicy: Always - containers: - - name: pgstac - image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} - args: - {{- toYaml .Values.db.command | nindent 12 }} - envFrom: - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - ports: - - containerPort: 5432 - resources: - limits: - cpu: {{ .Values.db.settings.resources.limits.cpu }} - memory: {{ .Values.db.settings.resources.limits.memory }} - requests: - cpu: {{ .Values.db.settings.resources.requests.cpu }} - memory: {{ .Values.db.settings.resources.requests.memory }} - volumeMounts: - - mountPath: /data - name: pgstac-claim-{{ $.Release.Name }} - {{- if .Values.db.enable_data_fixtures }} - - name: loader - image: {{ .Values.db.image.name }}:{{ .Values.db.image.tag }} - command: - - "sh" - args: - - "/opt/initdb/load.sh" - envFrom: - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - ports: - - containerPort: 6543 - volumeMounts: - - mountPath: /data - name: pgstac-claim-{{ $.Release.Name }} - - mountPath: /opt/initdb/sql-data - name: initdb-sql-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/json-data - name: initdb-json-volume-{{ $.Release.Name }} - - mountPath: /opt/initdb/ - name: initdb-sh-volume-{{ $.Release.Name }} - {{- end }} - volumes: - - name: pgstac-claim-{{ $.Release.Name }} - persistentVolumeClaim: - claimName: pgstac-claim-{{ $.Release.Name }} - - name: initdb-sql-volume-{{ $.Release.Name }} - configMap: - name: initdb-sql-config-{{ $.Release.Name }} - - name: initdb-json-volume-{{ $.Release.Name }} - configMap: - name: initdb-json-config-{{ $.Release.Name }} - - name: initdb-sh-volume-{{ $.Release.Name }} - configMap: - name: initdb-sh-config-{{ $.Release.Name }} -{{- end }} diff --git a/helm-chart/eoapi/templates/db/pvc.yaml b/helm-chart/eoapi/templates/db/pvc.yaml deleted file mode 100644 index 69e1ed4f..00000000 --- a/helm-chart/eoapi/templates/db/pvc.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.db.enabled }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pgstac-claim-{{ $.Release.Name }} -spec: - storageClassName: {{ .Values.db.storageClassName }} -{{- if .Values.db.accessModes }} - accessModes: -{{ toYaml .Values.db.accessModes | indent 4 }} -{{- end }} - resources: - requests: - storage: {{ .Values.db.settings.resources.requests.storage }} -{{- end }} \ No newline at end of file diff --git a/helm-chart/eoapi/templates/db/secrets.yaml b/helm-chart/eoapi/templates/db/secrets.yaml deleted file mode 100644 index fc48a9a1..00000000 --- a/helm-chart/eoapi/templates/db/secrets.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if .Values.db.enabled }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: pgstac-secrets-{{ $.Release.Name }} -type: "Opaque" -stringData: - {{- range $envKey, $envValue := .Values.db.settings.secrets }} - {{ upper $envKey }}: "{{ $envValue }}" - {{- /* stac-utils seems to require different environment variable for postgres so handle here via if/else to - avoid having to pass more arg secrets */ -}} - {{- if eq $envKey "PGPASSWORD" }} - POSTGRES_PASS: "{{ $envValue }}" - {{- end }} - {{- if eq $envKey "PGDATABASE" }} - POSTGRES_DBNAME: "{{ $envValue }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/helm-chart/eoapi/templates/db/service.yaml b/helm-chart/eoapi/templates/db/service.yaml deleted file mode 100644 index af8bcb42..00000000 --- a/helm-chart/eoapi/templates/db/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{{- if .Values.db.enabled }} ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: pgstac - name: pgstac -spec: - ports: - - name: "5432" - port: 5432 - targetPort: 5432 - selector: - app: pgstac -{{- end }} diff --git a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml b/helm-chart/eoapi/templates/pgstacbootstrap/configmap.yaml similarity index 95% rename from helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml rename to helm-chart/eoapi/templates/pgstacbootstrap/configmap.yaml index e82dcc6f..2459bf22 100644 --- a/helm-chart/eoapi/templates/pgstacboostrap/configmap.yaml +++ b/helm-chart/eoapi/templates/pgstacbootstrap/configmap.yaml @@ -15,7 +15,7 @@ data: pgstac-settings.sql: | {{ $.Files.Get "initdb-data/settings/pgstac-settings.sql" | nindent 4 }} --- -{{- if or .Values.pgstacBootstrap.settings.loadSamples (eq .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES "true") }} +{{- if .Values.pgstacBootstrap.settings.loadSamples }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/helm-chart/eoapi/templates/pgstacboostrap/job.yaml b/helm-chart/eoapi/templates/pgstacbootstrap/job.yaml similarity index 76% rename from helm-chart/eoapi/templates/pgstacboostrap/job.yaml rename to helm-chart/eoapi/templates/pgstacbootstrap/job.yaml index ec8cbeb3..4d47b5c5 100644 --- a/helm-chart/eoapi/templates/pgstacboostrap/job.yaml +++ b/helm-chart/eoapi/templates/pgstacbootstrap/job.yaml @@ -45,13 +45,9 @@ spec: - "-c" args: - | - # Set standard PG environment variables from secrets - export PGHOST=$POSTGRES_HOST - export PGPORT=$POSTGRES_PORT - export PGUSER=$POSTGRES_USER - export PGPASSWORD=$POSTGRES_PASS - export PGDATABASE=$POSTGRES_DBNAME - + # Database connection configured through standard PG* environment variables + # Environment variables are already set by the container + # Wait for the database to be ready echo "Waiting for database to be ready..." pypgstac pgready @@ -72,13 +68,13 @@ spec: {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} volumeMounts: - mountPath: /opt/settings - name: pgstac-settings-volume-{{ $.Release.Name }} + name: pgstac-settings-volume-{{ .Release.Name }} env: - {{ include "eoapi.pgstacSecrets" . | nindent 12 }} + {{- include "eoapi.postgresqlEnv" . | nindent 12 }} volumes: - - name: pgstac-settings-volume-{{ $.Release.Name }} + - name: pgstac-settings-volume-{{ .Release.Name }} configMap: - name: pgstac-settings-config-{{ $.Release.Name }} + name: pgstac-settings-config-{{ .Release.Name }} {{- with .Values.pgstacBootstrap.settings.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -90,7 +86,7 @@ spec: backoffLimit: 1 {{- end }} -{{- if and .Values.pgstacBootstrap.enabled (or .Values.pgstacBootstrap.settings.loadSamples (eq .Values.pgstacBootstrap.settings.envVars.LOAD_FIXTURES "true")) }} +{{- if and .Values.pgstacBootstrap.enabled .Values.pgstacBootstrap.settings.loadSamples }} --- apiVersion: batch/v1 kind: Job @@ -120,12 +116,8 @@ spec: # Exit immediately if a command exits with a non-zero status set -e - # Set standard PG environment variables from secrets - export PGHOST=$POSTGRES_HOST - export PGPORT=$POSTGRES_PORT - export PGUSER=$POSTGRES_USER - export PGPASSWORD=$POSTGRES_PASS - export PGDATABASE=$POSTGRES_DBNAME + # Database connection configured through standard PG* environment variables + # Environment variables are already set by the container # Load sample data echo "Loading sample collections..." @@ -142,18 +134,18 @@ spec: {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }} volumeMounts: - mountPath: /opt/sql - name: initdb-sql-volume-{{ $.Release.Name }} + name: initdb-sql-volume-{{ .Release.Name }} - mountPath: /opt/data - name: initdb-json-volume-{{ $.Release.Name }} + name: initdb-json-volume-{{ .Release.Name }} env: - {{ include "eoapi.pgstacSecrets" . | nindent 12 }} + {{- include "eoapi.postgresqlEnv" . | nindent 12 }} volumes: - - name: initdb-sql-volume-{{ $.Release.Name }} + - name: initdb-sql-volume-{{ .Release.Name }} configMap: - name: initdb-sql-config-{{ $.Release.Name }} - - name: initdb-json-volume-{{ $.Release.Name }} + name: initdb-sql-config-{{ .Release.Name }} + - name: initdb-json-volume-{{ .Release.Name }} configMap: - name: initdb-json-config-{{ $.Release.Name }} + name: initdb-json-config-{{ .Release.Name }} {{- with .Values.pgstacBootstrap.settings.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 25602129..43b75440 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -1,4 +1,4 @@ -{{ include "eoapi.validateTempDB" . }} +{{- include "eoapi.validatePostgresql" . }} {{- range $serviceName, $v := .Values -}} {{- if has $serviceName $.Values.apiServices }} {{- if index $v "enabled" }} @@ -74,20 +74,20 @@ spec: - containerPort: {{ $.Values.service.port }} resources: {{- toYaml (index $v "settings" "resources") | nindent 10 }} - {{- if $.Values.postgrescluster.enabled }} env: - {{- include "eoapi.pgstacSecrets" $ | nindent 12 }} - {{- end }} + {{- if or $.Values.postgrescluster.enabled (eq $.Values.postgresql.type "postgrescluster") }} + {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} + {{- else if eq $.Values.postgresql.type "external-plaintext" }} + {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} + {{- else if eq $.Values.postgresql.type "external-secret" }} + {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} + {{- end }} envFrom: # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. # we could just template them out here immediately with `value: $_` but this allows us # to store them in k8s intermediately and change them and then bounce deploys if needed - configMapRef: name: {{ $serviceName }}-envvar-configmap-{{ $.Release.Name }} - {{- if $.Values.db.enabled }} - - secretRef: - name: pgstac-secrets-{{ $.Release.Name }} - {{- end }} {{- if index $v "settings" "envSecrets" }} {{- range $secret := index $v "settings" "envSecrets" }} - secretRef: diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 9a5136dd..a5a6aaf3 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -56,47 +56,34 @@ comment_db: > The `postgrescluster` specs below are pass-through values to configure those separate charts. For more information read https://access.crunchydata.com/documentation/postgres-operator/latest -# DEPRECATED: this is the backward compatible way we originally did things -# and a temporary solution for EOEPCA. Since disabled by default most people SHOULD NOT -# use this option as it won't be talked about explicitly in the docs -db: - enabled: false - image: - name: ghcr.io/stac-utils/pgstac - tag: v0.9.5 - command: - - "postgres" - - "-N" - - "500" - # toggle to true||false if you want the db test fixtures loaded - enable_data_fixtures: true - storageClassName: "" - accessModes: - - ReadWriteOnce - settings: - resources: - requests: - storage: "100Mi" - cpu: "512m" - memory: "1024Mi" - limits: - cpu: "512m" - memory: "1024Mi" - secrets: - POSTGRES_DB: "postgis" - POSTGRES_USER: "" - POSTGRES_PASSWORD: "" - POSTGRES_PORT: "5432" - POSTGRES_HOST: "pgstac" - POSTGRES_HOST_READER: "pgstac" - POSTGRES_HOST_WRITER: "pgstac" - DB_MIN_CONN_SIZE: "1" - DB_MAX_CONN_SIZE: "15" - # default connect: https://www.postgresql.org/docs/current/libpq-envars.html - PGDATA: "/data/pgdata" - PGUSER: "" - PGPASSWORD: "" - PGDATABASE: "postgis" +# New unified PostgreSQL configuration +postgresql: + # Management type: "postgrescluster" (default), "external-plaintext", or "external-secret" + type: "postgrescluster" + + # Configuration for external PostgreSQL (used when type is "external-plaintext" or "external-secret") + external: + # Connection information + host: "" + port: "5432" + database: "eoapi" + + # Credentials configuration (used when type is "external-plaintext") + credentials: + username: "" + password: "" + + # Secret reference (used when type is "external-secret") + existingSecret: + name: "" + # Key mapping for the secret + keys: + username: "username" + password: "password" + # Optional: if these are provided in the secret + host: "host" + port: "port" + database: "database" # this is declared as a dependency of eoapi in helm-chart/eoapi/Chart.yaml postgrescluster: From 637dc9198b8805c2d54b2751e555a662cf7129a4 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Thu, 17 Apr 2025 14:22:57 +0200 Subject: [PATCH 2/4] Add PostgreSQL host reader and writer environment variables, and include DATABASE_URL for connection string --- helm-chart/eoapi/templates/_helpers.tpl | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/helm-chart/eoapi/templates/_helpers.tpl b/helm-chart/eoapi/templates/_helpers.tpl index 6abc43a7..f10ae5a0 100644 --- a/helm-chart/eoapi/templates/_helpers.tpl +++ b/helm-chart/eoapi/templates/_helpers.tpl @@ -128,6 +128,16 @@ PostgreSQL cluster secrets secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} key: host +- name: POSTGRES_HOST_READER + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: host +- name: POSTGRES_HOST_WRITER + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: host - name: POSTGRES_PASS valueFrom: secretKeyRef: @@ -138,6 +148,11 @@ PostgreSQL cluster secrets secretKeyRef: name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} key: dbname +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ $.Release.Name }}-pguser-{{ index $v "name" }} + key: uri {{- end }} {{- end }} - name: PGADMIN_URI @@ -169,10 +184,16 @@ External PostgreSQL with plaintext credentials value: {{ .Values.postgresql.external.port | quote }} - name: POSTGRES_HOST value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_HOST_READER + value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_HOST_WRITER + value: {{ .Values.postgresql.external.host | quote }} - name: POSTGRES_PASS value: {{ .Values.postgresql.external.credentials.password | quote }} - name: POSTGRES_DBNAME value: {{ .Values.postgresql.external.database | quote }} +- name: DATABASE_URL + value: "postgresql://{{ .Values.postgresql.external.credentials.username }}:{{ .Values.postgresql.external.credentials.password }}@{{ .Values.postgresql.external.host }}:{{ .Values.postgresql.external.port }}/{{ .Values.postgresql.external.database }}" {{- end }} {{/* @@ -214,11 +235,25 @@ External PostgreSQL with secret credentials secretKeyRef: name: {{ .Values.postgresql.external.existingSecret.name }} key: {{ .Values.postgresql.external.existingSecret.keys.host }} +- name: POSTGRES_HOST_READER + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.host }} +- name: POSTGRES_HOST_WRITER + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.host }} {{- else }} - name: PGHOST value: {{ .Values.postgresql.external.host | quote }} - name: POSTGRES_HOST value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_HOST_READER + value: {{ .Values.postgresql.external.host | quote }} +- name: POSTGRES_HOST_WRITER + value: {{ .Values.postgresql.external.host | quote }} {{- end }} {{- if .Values.postgresql.external.existingSecret.keys.port }} @@ -256,6 +291,18 @@ External PostgreSQL with secret credentials - name: POSTGRES_DBNAME value: {{ .Values.postgresql.external.database | quote }} {{- end }} + +# Add DATABASE_URL for connection string +{{- if .Values.postgresql.external.existingSecret.keys.uri }} +- name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ .Values.postgresql.external.existingSecret.name }} + key: {{ .Values.postgresql.external.existingSecret.keys.uri }} +{{- else }} +- name: DATABASE_URL + value: "postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST):$(PGPORT)/$(PGDATABASE)" +{{- end }} {{- end }} {{/* From ee41f4b7bfe133d8de1c34b95d6746cb3baa59a0 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Mon, 28 Apr 2025 17:30:40 +0200 Subject: [PATCH 3/4] Added a clarifying comment in values.yaml to explain that values in the external secret (host, port, database) will override the corresponding values defined in external.host, external.port, and external.database. Confirmed that the conditional blocks in deployment.yaml were already consolidated to eliminate redundancy. The file was already using a single include statement for PostgreSQL environment variables: env: {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} Removed the unused eoapi.mapLegacyPostgresql helper function from _helpers.tpl as it wasn't being referenced anywhere in the codebase. --- helm-chart/eoapi/templates/_helpers.tpl | 24 ------------------- .../eoapi/templates/services/deployment.yaml | 6 ----- helm-chart/eoapi/values.yaml | 1 + 3 files changed, 1 insertion(+), 30 deletions(-) diff --git a/helm-chart/eoapi/templates/_helpers.tpl b/helm-chart/eoapi/templates/_helpers.tpl index f10ae5a0..a8469620 100644 --- a/helm-chart/eoapi/templates/_helpers.tpl +++ b/helm-chart/eoapi/templates/_helpers.tpl @@ -344,30 +344,6 @@ Validate PostgreSQL configuration {{- end }} {{- end }} -{{/* -Map legacy configuration to new postgresql configuration -*/}} -{{- define "eoapi.mapLegacyPostgresql" -}} -{{- $postgresql := dict }} -{{- if .Values.postgrescluster.enabled }} - {{- $_ := set $postgresql "type" "postgrescluster" }} -{{- else if .Values.db.enabled }} - {{- $_ := set $postgresql "type" "external-plaintext" }} - {{- $external := dict }} - {{- $_ := set $external "host" .Values.db.settings.secrets.POSTGRES_HOST }} - {{- $_ := set $external "port" .Values.db.settings.secrets.POSTGRES_PORT }} - {{- $_ := set $external "database" .Values.db.settings.secrets.POSTGRES_DB }} - {{- $credentials := dict }} - {{- $_ := set $credentials "username" .Values.db.settings.secrets.POSTGRES_USER }} - {{- $_ := set $credentials "password" .Values.db.settings.secrets.POSTGRES_PASSWORD }} - {{- $_ := set $external "credentials" $credentials }} - {{- $_ := set $postgresql "external" $external }} -{{- else }} - {{- $_ := set $postgresql "type" "postgrescluster" }} -{{- end }} -{{- $postgresql | toYaml }} -{{- end }} - {{/* values.schema.json doesn't play nice combined value checks so we use this helper function to check autoscaling rules diff --git a/helm-chart/eoapi/templates/services/deployment.yaml b/helm-chart/eoapi/templates/services/deployment.yaml index 43b75440..afe02913 100644 --- a/helm-chart/eoapi/templates/services/deployment.yaml +++ b/helm-chart/eoapi/templates/services/deployment.yaml @@ -75,13 +75,7 @@ spec: resources: {{- toYaml (index $v "settings" "resources") | nindent 10 }} env: - {{- if or $.Values.postgrescluster.enabled (eq $.Values.postgresql.type "postgrescluster") }} {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} - {{- else if eq $.Values.postgresql.type "external-plaintext" }} - {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} - {{- else if eq $.Values.postgresql.type "external-secret" }} - {{- include "eoapi.postgresqlEnv" $ | nindent 12 }} - {{- end }} envFrom: # NOTE: there's no reason we need to use a `ConfigMap` or `Secret` here to get os env vars into the pod. # we could just template them out here immediately with `value: $_` but this allows us diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index a5a6aaf3..563fa3ea 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -81,6 +81,7 @@ postgresql: username: "username" password: "password" # Optional: if these are provided in the secret + # Note: These values override external.host, external.port and external.database if defined host: "host" port: "port" database: "database" From 0d34ea8b411a56204ca3c1a82706c7079ee8fdf6 Mon Sep 17 00:00:00 2001 From: Emmanuel Mathot Date: Tue, 29 Apr 2025 10:44:34 +0200 Subject: [PATCH 4/4] Refactor: Update comment for unified PostgreSQL configuration in values.yaml --- helm-chart/eoapi/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-chart/eoapi/values.yaml b/helm-chart/eoapi/values.yaml index 563fa3ea..d2ebc2d4 100644 --- a/helm-chart/eoapi/values.yaml +++ b/helm-chart/eoapi/values.yaml @@ -56,7 +56,7 @@ comment_db: > The `postgrescluster` specs below are pass-through values to configure those separate charts. For more information read https://access.crunchydata.com/documentation/postgres-operator/latest -# New unified PostgreSQL configuration +# unified PostgreSQL configuration postgresql: # Management type: "postgrescluster" (default), "external-plaintext", or "external-secret" type: "postgrescluster"