Merge pull request #2603 from lsst-sqre/tickets/DM-40947

DM-40947: Add secrets config for more applications

applications/obsloctap/README.md

@@ -11,6 +11,7 @@ Publish observing schedule
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | config.persistentVolumeClaims | list | `[]` | PersistentVolumeClaims to create. |
+| config.separateSecrets | bool | `false` | Whether to use the new secrets management scheme |
 | config.volume_mounts | list | `[]` | Mount points for additional volumes |
 | config.volumes | list | `[]` | Additional volumes to attach |
 | environment | object | `{}` | Environment variables (e.g. butler configuration/auth parms) for panel |

applications/obsloctap/secrets.yaml

@@ -0,0 +1,20 @@
+"aws-credentials.ini":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store, formatted using
+    AWS syntax for use with boto.
+  copy:
+    application: nublado
+    key: "aws-credentials.ini"
+"butler-gcs-idf-creds.json":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store in the native
+    Google syntax, containing the private asymmetric key.
+  copy:
+    application: nublado
+    key: "butler-gcs-idf-creds.json"
+"postgres-credentials.txt":
+  description: >-
+    PostgreSQL credentials in its pgpass format for the Butler database.
+  copy:
+    application: nublado
+    key: "postgres-credentials.txt"

applications/obsloctap/templates/deployment.yaml

@@ -18,7 +18,7 @@ spec:
         # butler-secrets-raw is the secrets we get from vault
         - name: "butler-secrets-raw"
           secret:
-            secretName: "butler-secret"
+            secretName: {{ include "obsloctap.fullname" . }}
         # butler-secrets are the copied and chmoded versions
         - name: "butler-secrets"
           emptyDir: {}

applications/obsloctap/templates/vault-secrets.yaml

@@ -1,10 +1,13 @@
----
 apiVersion: ricoberger.de/v1alpha1
 kind: VaultSecret
 metadata:
-  name: butler-secret
+  name: {{ template "obsloctap.fullname" . }}
   labels:
     {{- include "obsloctap.labels" . | nindent 4 }}
 spec:
+{{- if .Values.config.separateSecrets }}
+  path: "{{ .Values.global.vaultSecretsPath }}/obsloctap"
+{{- else }}
   path: "{{ .Values.global.vaultSecretsPath }}/butler-secret"
+{{- end }}
   type: Opaque

applications/obsloctap/values.yaml

@@ -16,7 +16,6 @@ ingress:
   # -- Additional annotations to add to the ingress
   annotations: {}
 
-
 config:
   # -- Additional volumes to attach
   volumes: []
@@ -27,6 +26,8 @@ config:
   # -- PersistentVolumeClaims to create.
   persistentVolumeClaims: []
 
+  # -- Whether to use the new secrets management scheme
+  separateSecrets: false
 
 # The following will be set by parameters injected by Argo CD and should not
 # be set in the individual environment values files.

applications/plot-navigator/README.md

@@ -11,12 +11,13 @@ Panel-based plot viewer
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | config.persistentVolumeClaims | list | `[]` | PersistentVolumeClaims to create. |
+| config.separateSecrets | bool | `false` | Whether to use the new secrets management scheme |
 | config.volume_mounts | list | `[]` | Mount points for additional volumes |
 | config.volumes | list | `[]` | Additional volumes to attach |
 | environment | object | `{}` | Environment variables (e.g. butler configuration/auth parms) for panel |
 | global.baseUrl | string | Set by Argo CD | Base URL for the environment |
 | global.host | string | Set by Argo CD | Host name for ingress |
 | global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |
 | image.repository | string | `"ghcr.io/lsst-dm/pipetask-plot-navigator"` | plot-navigator image to use |
-| image.tag | string | `""` |  |
+| image.tag | string | The appVersion of the chart | Tag of plot-navigator image to use |
 | ingress.annotations | object | `{}` | Additional annotations to add to the ingress |

applications/plot-navigator/secrets.yaml

@@ -0,0 +1,20 @@
+"aws-credentials.ini":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store, formatted using
+    AWS syntax for use with boto.
+  copy:
+    application: nublado
+    key: "aws-credentials.ini"
+"butler-gcs-idf-creds.json":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store in the native
+    Google syntax, containing the private asymmetric key.
+  copy:
+    application: nublado
+    key: "butler-gcs-idf-creds.json"
+"postgres-credentials.txt":
+  description: >-
+    PostgreSQL credentials in its pgpass format for the Butler database.
+  copy:
+    application: nublado
+    key: "postgres-credentials.txt"

applications/plot-navigator/templates/deployment.yaml

@@ -14,13 +14,11 @@ spec:
       labels:
         {{- include "plot-navigator.selectorLabels" . | nindent 8 }}
     spec:
-      imagePullSecrets:
-        - name: "pull-secret"
       volumes:
         # butler-secrets-raw is the secrets we get from vault
         - name: "butler-secrets-raw"
           secret:
-            secretName: "butler-secret"
+            secretName: {{ include "plot-navigator.fullname" . }}
         # butler-secrets are the copied and chmoded versions
         - name: "butler-secrets"
           emptyDir: {}

applications/plot-navigator/templates/vault-secrets.yaml

@@ -1,20 +1,13 @@
----
 apiVersion: ricoberger.de/v1alpha1
 kind: VaultSecret
 metadata:
-  name: butler-secret
+  name: {{ template "plot-navigator.fullname" . }}
   labels:
     {{- include "plot-navigator.labels" . | nindent 4 }}
 spec:
+{{- if .Values.config.separateSecrets }}
+  path: "{{ .Values.global.vaultSecretsPath }}/plot-navigator"
+{{- else }}
   path: "{{ .Values.global.vaultSecretsPath }}/butler-secret"
+{{- end }}
   type: Opaque
----
-apiVersion: ricoberger.de/v1alpha1
-kind: VaultSecret
-metadata:
-  name: pull-secret
-  labels:
-    {{- include "plot-navigator.labels" . | nindent 4 }}
-spec:
-  path: "{{- .Values.global.vaultSecretsPath }}/pull-secret"
-  type: kubernetes.io/dockerconfigjson

applications/plot-navigator/values.yaml

@@ -1,6 +1,9 @@
 image:
   # -- plot-navigator image to use
   repository: ghcr.io/lsst-dm/pipetask-plot-navigator
+
+  # -- Tag of plot-navigator image to use
+  # @default -- The appVersion of the chart
   tag: ""
 
 # -- Environment variables (e.g. butler configuration/auth parms) for panel
@@ -10,7 +13,6 @@ ingress:
   # -- Additional annotations to add to the ingress
   annotations: {}
 
-
 config:
   # -- Additional volumes to attach
   volumes: []
@@ -21,6 +23,8 @@ config:
   # -- PersistentVolumeClaims to create.
   persistentVolumeClaims: []
 
+  # -- Whether to use the new secrets management scheme
+  separateSecrets: false
 
 # The following will be set by parameters injected by Argo CD and should not
 # be set in the individual environment values files.

applications/production-tools/README.md

@@ -11,14 +11,15 @@ A collection of utility pages for monitoring data processing.
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity rules for the production-tools deployment pod |
+| config.separateSecrets | bool | `false` | Whether to use the new secrets management scheme |
 | environment | object | `{}` |  |
 | fullnameOverride | string | `""` | Override the full name for resources (includes the release name) |
 | global.baseUrl | string | Set by Argo CD | Base URL for the environment |
 | global.host | string | Set by Argo CD | Host name for ingress |
 | global.vaultSecretsPath | string | Set by Argo CD | Base path for Vault secrets |
 | image.pullPolicy | string | `"IfNotPresent"` | Pull policy for the production-tools image |
 | image.repository | string | `"lsstdm/production_tools"` | Image to use in the production-tools deployment |
-| image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. |
+| image.tag | string | The appVersion of the chart | Tag of production-tools image to use |
 | ingress.annotations | object | `{}` | Additional annotations for the ingress rule |
 | nameOverride | string | `""` | Override the base name for resources |
 | nodeSelector | object | `{}` | Node selection rules for the production-tools deployment pod |

applications/production-tools/secrets.yaml

@@ -0,0 +1,20 @@
+"aws-credentials.ini":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store, formatted using
+    AWS syntax for use with boto.
+  copy:
+    application: nublado
+    key: "aws-credentials.ini"
+"butler-gcs-idf-creds.json":
+  description: >-
+    Google Cloud Storage credentials to the Butler data store in the native
+    Google syntax, containing the private asymmetric key.
+  copy:
+    application: nublado
+    key: "butler-gcs-idf-creds.json"
+"postgres-credentials.txt":
+  description: >-
+    PostgreSQL credentials in its pgpass format for the Butler database.
+  copy:
+    application: nublado
+    key: "postgres-credentials.txt"

applications/production-tools/templates/deployment.yaml

@@ -29,7 +29,7 @@ spec:
         # butler-secrets-raw is the secrets we get from vault
         - name: "butler-secrets-raw"
           secret:
-            secretName: "butler-secret"
+            secretName: {{ include "production-tools.fullname" . }}
         # butler-secrets are the copied and chmoded versions
         - name: "butler-secrets"
           emptyDir: {}

applications/production-tools/templates/vault-secrets.yaml

@@ -1,12 +1,15 @@
----
 apiVersion: ricoberger.de/v1alpha1
 kind: VaultSecret
 metadata:
-  name: butler-secret
+  name: {{ template "production-tools.fullname" . }}
   labels:
     {{- include "production-tools.labels" . | nindent 4 }}
 spec:
+{{- if .Values.config.separateSecrets }}
+  path: "{{ .Values.global.vaultSecretsPath }}/production-tools"
+{{- else }}
   path: "{{ .Values.global.vaultSecretsPath }}/butler-secret"
+{{- end }}
   type: Opaque
 ---
 apiVersion: ricoberger.de/v1alpha1

applications/production-tools/values.yaml

@@ -12,7 +12,8 @@ image:
   # -- Pull policy for the production-tools image
   pullPolicy: IfNotPresent
 
-  # -- Overrides the image tag whose default is the chart appVersion.
+  # -- Tag of production-tools image to use
+  # @default -- The appVersion of the chart
   tag: ""
 
 # -- Override the base name for resources
@@ -31,6 +32,10 @@ ingress:
   # -- Additional annotations for the ingress rule
   annotations: {}
 
+config:
+  # -- Whether to use the new secrets management scheme
+  separateSecrets: false
+
 # -- Resource limits and requests for the production-tools deployment pod
 resources: {}
 

docs/admin/migrating-secrets.rst

@@ -194,6 +194,16 @@ Switch to the new secrets tree
    If you are using a static secrets file, add the ``--secrets`` flag pointing to that file.
    This will fix any secrets that are missing or incorrect in Vault.
 
+#. Some Phalanx applications need to know whether the old or new secrets layout is in use.
+   On your working branch, add the necessary settings for those applications to their :file:`values-{environment}.yaml` files for your environment.
+   Applications to review:
+
+   - :px-app:`datalinker` (``config.separateSecrets``)
+   - :px-app:`nublado` (``secrets.templateSecrets``)
+   - :px-app:`obsloctap` (``config.separateSecrets``)
+   - :px-app:`plot-navigator` (``config.separateSecrets``)
+   - :px-app:`production-tools` (``config.separateSecrets``)
+
 #. You're now ready to test the new secrets tree.
    You can do this on a branch that contains the changes you made above.
 

tests/config_test.py

@@ -10,10 +10,7 @@
     "giftless",
     "linters",
     "monitoring",
-    "obsloctap",
     "next-visit-fan-out",
-    "plot-navigator",
-    "production-tools",
 )
 """Temporary whitelist of applications that haven't added secrets.yaml."""