CORE-19372 Use databases for cluster schema (#5643)

This PR removes the concept of a "cluster DB". Instead, the databases stanza introduced for state managers can now also be used for the config, crypto, RBAC, and virtual node databases. By default, they will all share a database as before but they can be split apart. The other, perhaps more significant, change, is that the automatic bootstrap will now create separate users for each of crypto, RBAC, and virtual node databases, and one for worker that accesses the config DB. It is, therefore, now only necessary to give a single bootstrap user for each database and everything else is handled by the chart.
corda · Feb 9, 2024 · 69d474e · 69d474e
1 parent 2aa26d9
commit 69d474e
Show file tree

Hide file tree

Showing 36 changed files with 1,133 additions and 1,655 deletions.
diff --git a/.ci/e2eTests/Jenkinsfile b/.ci/e2eTests/Jenkinsfile
@@ -1,4 +1,4 @@
-@Library('corda-shared-build-pipeline-steps@5.2') _
+@Library('corda-shared-build-pipeline-steps@cluster-db/CORE-19372') _
 
 endToEndPipeline(
     assembleAndCompile: false,

diff --git a/.ci/e2eTests/corda.yaml b/.ci/e2eTests/corda.yaml
@@ -4,33 +4,11 @@ databases:
     host: null  # passed in from jenkins pipeline
     port: 5432
     type: "postgresql"
-    username:
-      value: "user"
-      valueFrom:
-        secretKeyRef:
-          name: ""
-          key: ""
-    password:
-      valueFrom:
-        secretKeyRef:
-          key: "password"
-          name: "" # passed in from jenkins pipeline
   - id: "isolated-state-manager"
     name: "state-manager"
     host: null # passed in from jenkins pipeline
     port: 5432
     type: "postgresql"
-    username:
-      value: "state-manager-user"
-      valueFrom:
-        secretKeyRef:
-          name: ""
-          key: ""
-    password:
-      valueFrom:
-        secretKeyRef:
-          key: "password"
-          name: "" # passed in from jenkins pipeline
 
 # Fields 'type' and 'partition' already have defaults set by the Helm Chart, adding them here just for clarity
 stateManager:
@@ -61,21 +39,10 @@ stateManager:
 
 bootstrap:
   db:
-    cluster:
-      username:
-        value: "postgres"
-      password:
-        valueFrom:
-          secretKeyRef:
-            key: "postgres-password"
     databases:
       - id: "default"
         username:
           value: "postgres"
-          valueFrom:
-            secretKeyRef:
-              name: ""
-              key: ""
         password:
           valueFrom:
             secretKeyRef:
@@ -84,10 +51,6 @@ bootstrap:
       - id: "isolated-state-manager"
         username:
           value: "state-manager-user"
-          valueFrom:
-            secretKeyRef:
-              name: ""
-              key: ""
         password:
           valueFrom:
             secretKeyRef:
@@ -107,15 +70,6 @@ bootstrap:
 logging:
   format: "text"
 
-db:
-  cluster:
-    password:
-      valueFrom:
-        secretKeyRef:
-          key: "password"
-    username:
-      value: "user"
-
 kafka:
   bootstrapServers: "prereqs-kafka:9092"
   tls:

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,4 +1,4 @@
-@Library('corda-shared-build-pipeline-steps@5.2') _
+@Library('corda-shared-build-pipeline-steps@cluster-db/CORE-19372') _
 
 cordaPipelineKubernetesAgent(
     dailyBuildCron: 'H H/6 * * *',

diff --git a/charts/corda-lib/templates/_bootstrap.tpl b/charts/corda-lib/templates/_bootstrap.tpl
diff --git a/charts/corda-lib/templates/_database.tpl b/charts/corda-lib/templates/_database.tpl
@@ -15,13 +15,139 @@
 
 
 {{/*
-    Default Name for Secrets Containing Database Credentials
-    The resulting secret name is "chartName-runtime-databaseId-db"
+    Environment variables (*_DB_USERNAME and *_DB_PASSWORD) to be used when configuring databases
 */}}
-{{- define "corda.db.runtimeCredentialsSecretName" -}}
+{{- define "corda.db.runtimeEnvironment" -}}
 {{- $ := index . 0 -}}
-{{- $dbId := index . 1 -}}
-{{ printf "%s-runtime-%s-db" ( include "corda.fullname" $ ) $dbId }}
+{{- $db := index . 1 -}}
+{{- $bootstrapSettings := index . 2 -}}
+- name: {{ upper ( snakecase $db ) }}_DB_USERNAME
+  valueFrom:
+    secretKeyRef:
+      {{- if (($bootstrapSettings.username.valueFrom).secretKeyRef).name }}
+      name: {{ $bootstrapSettings.username.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify bootstrap.db.%s.username.valueFrom.secretKeyRef.key" $db ) $bootstrapSettings.username.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ printf "%s-%s-db" ( include "corda.fullname" $ ) ( kebabcase $db ) }}
+      key: "username"
+      {{-   end }}
+- name: {{ upper ( snakecase $db ) }}_DB_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      {{- if (($bootstrapSettings.password.valueFrom).secretKeyRef).name }}
+      name: {{ $bootstrapSettings.password.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify bootstrap.db.%s.password.valueFrom.secretKeyRef.key" $db ) $bootstrapSettings.password.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ printf "%s-%s-db" ( include "corda.fullname" $ ) ( kebabcase $db ) }}
+      key: "password"
+      {{-   end }}
+{{- end -}}
+
+
+{{/*
+    Environment variables (CONFIG_*_DB_USERNAME and CONFIG_*_DB_PASSWORD) to be used when configuring config database
+    for each worker
+*/}}
+{{- define "corda.db.runtimeConfigEnvironment" -}}
+{{- $ := index . 0 -}}
+{{- range $workerName, $workerValues := $.Values.workers }}
+{{-   if $workerValues.config }}
+{{-     $configValues := $workerValues.config }}
+- name: CONFIG_{{ upper ( snakecase $workerName ) }}_DB_USERNAME
+  valueFrom:
+    secretKeyRef:
+      {{- if (($configValues.username.valueFrom).secretKeyRef).name }}
+      name: {{ $configValues.username.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify workers.%s.config.username.valueFrom.secretKeyRef.key" $workerName ) $configValues.username.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.workerCredentialsSecretName" ( list $ $workerName ) }}
+      key: "username"
+      {{-   end }}
+- name: CONFIG_{{ upper ( snakecase $workerName ) }}_DB_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      {{- if (($configValues.password.valueFrom).secretKeyRef).name }}
+      name: {{ $configValues.password.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify workers.%s.config.password.valueFrom.secretKeyRef.key" $workerName ) $configValues.password.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.workerCredentialsSecretName" ( list $ $workerName ) }}
+      key: "password"
+      {{-   end }}
+{{-   end -}}
+{{- end -}}
+{{- end -}}
+
+
+{{/*
+    Environment variables (BOOTSTRAP_*_DB_USERNAME and BOOTSTRAP_*_DB_PASSWORD) to be used when bootstrapping databases
+*/}}
+{{- define "corda.db.bootstrapEnvironment" -}}
+{{- $ := index . 0 -}}
+{{- $db := index . 1 -}}
+{{- $dbId := index . 2 -}}
+{{- $bootstrapSettings := index . 3 -}}
+- name: BOOTSTRAP_{{ upper ( snakecase $db ) }}_DB_USERNAME
+  valueFrom:
+    secretKeyRef:
+      {{- if (($bootstrapSettings.username.valueFrom).secretKeyRef).name }}
+      name: {{ $bootstrapSettings.username.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify bootstrap username.valueFrom.secretKeyRef.key for database '%s'" $dbId ) $bootstrapSettings.username.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.bootstrapCredentialsSecretName" ( list $ $dbId ) | quote }}
+      key: "username"
+      {{-   end }}
+- name: BOOTSTRAP_{{ upper ( snakecase $db ) }}_DB_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      {{- if (($bootstrapSettings.password.valueFrom).secretKeyRef).name }}
+      name: {{ $bootstrapSettings.password.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify bootstrap password.valueFrom.secretKeyRef.key for database '%s'" $dbId ) $bootstrapSettings.password.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.bootstrapCredentialsSecretName" ( list $ $dbId ) | quote }}
+      key: "password"
+      {{-   end }}
+{{- end -}}
+
+
+{{/*
+    Environment variables (CONFIG_DB_USERNAME and CONFIG_DB_PASSWORD) to be used when connecting to config database
+*/}}
+{{- define "corda.db.workerConfigEnvironment" -}}
+{{- $ := index . 0 -}}
+{{- $workerName := index . 1 -}}
+{{- $config := index . 2 -}}
+{{- $dbId := $config.storageId -}}
+- name: CONFIG_DB_USERNAME
+  valueFrom:
+    secretKeyRef:
+      {{- if (($config.username.valueFrom).secretKeyRef).name }}
+      name: {{ $config.username.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify username.valueFrom.secretKeyRef.key for database '%s'" $dbId ) $config.username.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.workerCredentialsSecretName" ( list $ $workerName ) | quote }}
+      key: "username"
+      {{-   end }}
+- name: CONFIG_DB_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      {{- if (($config.password.valueFrom).secretKeyRef).name }}
+      name: {{ $config.password.valueFrom.secretKeyRef.name | quote }}
+      key: {{ required ( printf "Must specify password.valueFrom.secretKeyRef.key for database '%s'" $dbId ) $config.password.valueFrom.secretKeyRef.key | quote }}
+      {{-   else }}
+      name: {{ include "corda.db.workerCredentialsSecretName" ( list $ $workerName ) | quote }}
+      key: "password"
+      {{-   end }}
+{{- end -}}
+
+
+{{/*
+    Default Name for Secrets Containing Database Credentials for a worker
+    The resulting secret name is "chartName-worker-workerName-db"
+*/}}
+{{- define "corda.db.workerCredentialsSecretName" -}}
+{{- $ := index . 0 -}}
+{{- $workerName := index . 1 -}}
+{{ printf "%s-%s-worker-db" ( include "corda.fullname" $ ) ( kebabcase $workerName ) }}
 {{- end -}}
 
 
@@ -60,7 +186,30 @@ jdbc:{{- .type -}}://{{- required ( printf "Must specify a host for database '%s
 {{-   end -}}
 {{- end -}}
 {{- if not $databaseFound -}}
-{{-   fail ( printf "Undefined persistent storage '%s' detected at %s" $dbId $reference ) -}}
+{{-   fail ( printf "Persistent storage '%s' referenced at %s undefined in databases" $dbId $reference ) -}}
+{{- end -}}
+{{ $defaultDatabaseConfig | toYaml }}
+{{- end -}}
+
+
+{{/*
+    Iterate through configured bootstrap databases and return the one matching the requested 'storageId'.
+    If a database with the requested 'storageId' can not be found, immediately fail the rendering process.
+*/}}
+{{- define "corda.db.bootstrapConfiguration" -}}
+{{- $ := index . 0 -}}
+{{- $dbId := index . 1 -}}
+{{- $reference := index . 2 -}}
+{{- $databaseFound := false -}}
+{{- $defaultDatabaseConfig := dict -}}
+{{- range $.Values.bootstrap.db.databases -}}
+{{-   if eq .id $dbId -}}
+{{-     $databaseFound = true -}}
+{{-     $defaultDatabaseConfig = . -}}
+{{-   end -}}
+{{- end -}}
+{{- if not $databaseFound -}}
+{{-   fail ( printf "Persistent storage '%s' referenced at %s undefined in bootstrap.db.databases" $dbId $reference ) -}}
 {{- end -}}
 {{ $defaultDatabaseConfig | toYaml }}
 {{- end -}}