[node] add support for extraDerivation for existingSecrets.

charts/node/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: node
 description: A Helm chart to deploy Substrate/Polkadot nodes
 type: application
-version: 5.10.0
+version: 5.11.0
 maintainers:
   - name: Parity
     url: https://github.com/paritytech/helm-charts

charts/node/README.md

@@ -18,7 +18,7 @@ This is intended behaviour. Make sure to run `git add -A` once again to stage ch
 
 # Substrate/Polkadot node Helm chart
 
-![Version: 5.10.0](https://img.shields.io/badge/Version-5.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 5.11.0](https://img.shields.io/badge/Version-5.11.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 ## Overview
 The Polkadot Helm Chart provides a convenient way to deploy and manage a Polkadot blockchain node in a Kubernetes cluster.
@@ -355,7 +355,7 @@ If you're running a collator node:
 | jaegerAgent.ports.samplingPort | HTTP | `5778` | serve configs, sampling strategies |
 | jaegerAgent.resources | object | `{}` | Resource limits & requests |
 | nameOverride | string | `""` | Provide a name in place of node for `app:` labels |
-| node | object | `{"allowUnsafeRpcMethods":false,"chain":"polkadot","chainData":{"annotations":{},"chainPath":null,"chainSnapshot":{"enabled":false,"filelistName":"files.txt","method":"gcs","url":""},"database":"rocksdb","ephemeral":{"enabled":false,"type":"emptyDir"},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"pruning":1000,"storageClass":"","volumeSize":"100Gi"},"chainKeystore":{"accessModes":["ReadWriteOnce"],"annotations":{},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"mountInMemory":{"enabled":false,"sizeLimit":null},"storageClass":"","volumeSize":"10Mi"},"collatorExternalRelayChain":{"enabled":false,"relayChainRpcUrls":[]},"collatorLightClient":{"enabled":false,"relayChain":"","relayChainCustomChainspec":false,"relayChainCustomChainspecPath":"/chain-data/relay_chain_chainspec.json","relayChainCustomChainspecUrl":null},"collatorRelayChain":{"chain":"polkadot","chainData":{"annotations":{},"chainPath":"","chainSnapshot":{"enabled":false,"filelistName":"files.txt","method":"gcs","url":""},"database":"rocksdb","ephemeral":{"enabled":false,"type":"emptyDir"},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"pruning":1000,"storageClass":"","volumeSize":"100Gi"},"chainKeystore":{"accessModes":["ReadWriteOnce"],"annotations":{},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"mountInMemory":{"enabled":false,"sizeLimit":null},"storageClass":"","volumeSize":"10Mi"},"customChainspec":false,"customChainspecPath":"/relaychain-data/relay_chain_chainspec.json","customChainspecUrl":null,"flags":[],"prometheus":{"enabled":false,"port":9625}},"command":"polkadot","customChainspec":false,"customChainspecPath":"/chain-data/chainspec.json","customChainspecUrl":null,"customNodeKey":[],"enableOffchainIndexing":false,"enableSidecarLivenessProbe":false,"enableSidecarReadinessProbe":false,"enableStartupProbe":true,"existingSecrets":{"keys":[],"nodeKey":{}},"extraConfigmapMounts":[],"extraEnvVars":[],"extraSecretMounts":[],"flags":[],"forceDownloadChainspec":false,"isParachain":false,"keys":{},"legacyRpcFlags":false,"logLevels":[],"perNodeServices":{"apiService":{"annotations":{},"enabled":true,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"httpPort":9933,"prometheusPort":9615,"relayChainPrometheusPort":9625,"rpcPort":9944,"type":"ClusterIP","wsPort":9955},"paraP2pService":{"annotations":{},"enabled":false,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"port":30334,"publishUnreadyAddresses":true,"type":"NodePort","ws":{"enabled":false,"port":30335}},"relayP2pService":{"annotations":{},"enabled":false,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"port":30333,"publishUnreadyAddresses":true,"type":"NodePort","ws":{"enabled":false,"port":30334}},"setPublicAddressToExternalIp":{"autodiscoveryFix":false,"enabled":false,"ipRetrievalServiceUrl":"https://ifconfig.io"}},"persistGeneratedNodeKey":false,"persistentVolumeClaimRetentionPolicy":null,"podManagementPolicy":null,"prometheus":{"enabled":true,"port":9615},"replicas":1,"resources":{},"role":"full","serviceAnnotations":{},"serviceExtraPorts":[],"serviceMonitor":{"enabled":false,"interval":"30s","metricRelabelings":[],"namespace":null,"relabelings":[],"scrapeTimeout":"10s","targetLabels":["node"]},"startupProbeFailureThreshold":30,"substrateApiSidecar":{"enabled":false},"telemetryUrls":[],"tracing":{"enabled":false},"updateStrategy":{"enabled":false,"maxUnavailable":1,"type":"RollingUpdate"},"vault":{"authConfigServiceAccount":null,"authConfigType":null,"authPath":null,"authRole":null,"authType":null,"keys":{},"nodeKey":{}},"wasmRuntimeOverridesPath":"/chain-data/runtimes","wasmRuntimeUrl":""}` | Deploy a substrate node. ref: https://docs.substrate.io/tutorials/v3/private-network/ |
+| node | object | `{"allowUnsafeRpcMethods":false,"chain":"polkadot","chainData":{"annotations":{},"chainPath":null,"chainSnapshot":{"enabled":false,"filelistName":"files.txt","method":"gcs","url":""},"database":"rocksdb","ephemeral":{"enabled":false,"type":"emptyDir"},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"pruning":1000,"storageClass":"","volumeSize":"100Gi"},"chainKeystore":{"accessModes":["ReadWriteOnce"],"annotations":{},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"mountInMemory":{"enabled":false,"sizeLimit":null},"storageClass":"","volumeSize":"10Mi"},"collatorExternalRelayChain":{"enabled":false,"relayChainRpcUrls":[]},"collatorLightClient":{"enabled":false,"relayChain":"","relayChainCustomChainspec":false,"relayChainCustomChainspecPath":"/chain-data/relay_chain_chainspec.json","relayChainCustomChainspecUrl":null},"collatorRelayChain":{"chain":"polkadot","chainData":{"annotations":{},"chainPath":"","chainSnapshot":{"enabled":false,"filelistName":"files.txt","method":"gcs","url":""},"database":"rocksdb","ephemeral":{"enabled":false,"type":"emptyDir"},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"pruning":1000,"storageClass":"","volumeSize":"100Gi"},"chainKeystore":{"accessModes":["ReadWriteOnce"],"annotations":{},"kubernetesVolumeSnapshot":null,"kubernetesVolumeToClone":null,"mountInMemory":{"enabled":false,"sizeLimit":null},"storageClass":"","volumeSize":"10Mi"},"customChainspec":false,"customChainspecPath":"/relaychain-data/relay_chain_chainspec.json","customChainspecUrl":null,"flags":[],"prometheus":{"enabled":false,"port":9625}},"command":"polkadot","customChainspec":false,"customChainspecPath":"/chain-data/chainspec.json","customChainspecUrl":null,"customNodeKey":[],"enableOffchainIndexing":false,"enableSidecarLivenessProbe":false,"enableSidecarReadinessProbe":false,"enableStartupProbe":true,"existingSecrets":{"extraDerivation":"","keys":[],"nodeKey":{}},"extraConfigmapMounts":[],"extraEnvVars":[],"extraSecretMounts":[],"flags":[],"forceDownloadChainspec":false,"isParachain":false,"keys":{},"legacyRpcFlags":false,"logLevels":[],"perNodeServices":{"apiService":{"annotations":{},"enabled":true,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"httpPort":9933,"prometheusPort":9615,"relayChainPrometheusPort":9625,"rpcPort":9944,"type":"ClusterIP","wsPort":9955},"paraP2pService":{"annotations":{},"enabled":false,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"port":30334,"publishUnreadyAddresses":true,"type":"NodePort","ws":{"enabled":false,"port":30335}},"relayP2pService":{"annotations":{},"enabled":false,"externalDns":{"customPrefix":"","enabled":false,"hostname":"example.com","ttl":300},"externalTrafficPolicy":"Cluster","extraPorts":[],"port":30333,"publishUnreadyAddresses":true,"type":"NodePort","ws":{"enabled":false,"port":30334}},"setPublicAddressToExternalIp":{"autodiscoveryFix":false,"enabled":false,"ipRetrievalServiceUrl":"https://ifconfig.io"}},"persistGeneratedNodeKey":false,"persistentVolumeClaimRetentionPolicy":null,"podManagementPolicy":null,"prometheus":{"enabled":true,"port":9615},"replicas":1,"resources":{},"role":"full","serviceAnnotations":{},"serviceExtraPorts":[],"serviceMonitor":{"enabled":false,"interval":"30s","metricRelabelings":[],"namespace":null,"relabelings":[],"scrapeTimeout":"10s","targetLabels":["node"]},"startupProbeFailureThreshold":30,"substrateApiSidecar":{"enabled":false},"telemetryUrls":[],"tracing":{"enabled":false},"updateStrategy":{"enabled":false,"maxUnavailable":1,"type":"RollingUpdate"},"vault":{"authConfigServiceAccount":null,"authConfigType":null,"authPath":null,"authRole":null,"authType":null,"keys":{},"nodeKey":{}},"wasmRuntimeOverridesPath":"/chain-data/runtimes","wasmRuntimeUrl":""}` | Deploy a substrate node. ref: https://docs.substrate.io/tutorials/v3/private-network/ |
 | node.allowUnsafeRpcMethods | bool | `false` | Allow executing unsafe RPC methods |
 | node.chain | string | `"polkadot"` | Name of the chain |
 | node.chainData.annotations | object | `{}` | Annotations to add to the volumeClaimTemplates |
@@ -433,9 +433,9 @@ If you're running a collator node:
 | node.enableSidecarLivenessProbe | bool | `false` | Enable Node liveness probe through `paritytech/ws-health-exporter` running as a sidecar container |
 | node.enableSidecarReadinessProbe | bool | `false` | Enable Node readiness probe through `paritytech/ws-health-exporter` running as a sidecar container |
 | node.enableStartupProbe | bool | `true` | Enable Node container's startup probe |
-| node.existingSecrets | object | `{"keys":[],"nodeKey":{}}` | Inject keys from already existing Kubernetes secrets |
-| node.existingSecrets.keys | list | `[]` | List of kubernetes secret names to be added to the keystore. Each secret should contain 3 keys: type, scheme and seed Supercedes node.vault.keys |
-| node.existingSecrets.nodeKey | object | `{}` | K8s secret with node key Supercedes node.vault.nodeKey |
+| node.existingSecrets | object | `{"extraDerivation":"","keys":[],"nodeKey":{}}` | Inject keys from already existing Kubernetes secrets |
+| node.existingSecrets.keys | list | `[]` | List of kubernetes secret names to be added to the keystore. Each secret should contain 3 keys: type, scheme and seed Secret example: templates/keys.yaml Supercedes node.vault.keys |
+| node.existingSecrets.nodeKey | object | `{}` | K8s secret with node key Secret example: templates/customNodeKeySecret.yaml Supercedes node.vault.nodeKey |
 | node.extraConfigmapMounts | list | `[]` | Mount already existing ConfigMaps into the main container. https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#populate-a-volume-with-data-stored-in-a-configmap |
 | node.extraEnvVars | list | `[]` | Environment variables to set for the main container: |
 | node.extraSecretMounts | list | `[]` | Mount already existing k8s Secrets into main container. https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-files-from-a-pod NOTE: This is NOT used to inject keys to the keystore or add node key. |

charts/node/examples/local-rococo/README.md

@@ -85,3 +85,15 @@ helm delete bootnode validators parachain
 # Clean PVCs if needed
 # kubectl delete pvc --all
 ```
+
+### Advanced
+Deploy parachain by using  `existingSecrets` option. 
+1. Create secrets with node key(ID) and session keys. 
+```shell
+kubectl apply -f ./examples/local-rococo/secret.yaml
+```
+2. Deploy second parachain.
+```
+helm upgrade --install parachain2  . -f examples/local-rococo/parachain2.yaml
+```
+3. Onboard the parachain by following the previous steps.

charts/node/examples/local-rococo/parachain2.yaml

@@ -0,0 +1,72 @@
+image:
+  repository: parity/polkadot-parachain
+  tag: latest
+  pullPolicy: Always
+
+node:
+  chain: bridge-hub-rococo-local
+  command: polkadot-parachain
+  role: collator
+  replicas: 2
+  chainData:
+    pruning: 1000
+    storageClass: ""
+  chainKeystore:
+    storageClass: ""
+  existingSecrets:
+    keys:
+      - my-key-aura
+    extraDerivation: '$([ "${HOSTNAME##*-}" = "0" ] && echo "//Alice" || echo "//Bob")'
+    nodeKey:
+     secretName: my-node-key
+     secretKey: custom-node-key
+     appendPodIndex: true
+  flags:
+    - "--bootnodes /dns/parachain2-node-0/tcp/30334/p2p/12D3KooWF4B55vTeXa7g88nUdSHVk6DiRXTrsfQyJcg7TrXcCK8U"
+  isParachain: true
+  collatorRelayChain:
+    chain: rococo-local
+    customChainspecUrl: http://bootnode:8080/chainspec.json
+    forceDownloadChainspec: true
+    chainData:
+      storageClass: ""
+    flags:
+      - "--allow-private-ipv4"
+      - "--discover-local"
+
+extraInitContainers:
+  - name: dump-state-and-wasm
+    image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+    securityContext:
+      runAsUser: 0
+    command: [ "/bin/bash" ]
+    args:
+      - -c
+      - |
+        if [ "${HOSTNAME##*-}" = "0" ]; then
+          echo "Parachain Id:"
+          {{ .Values.node.command }} build-spec --chain {{ .Values.node.chain }} | grep -E 'para_id|parachainId' 
+          echo "Genesis head:"
+          {{ .Values.node.command }} export-genesis-state --chain {{ .Values.node.chain }}
+          echo ""
+          echo "Genesis wasm (validationCode) stored in /chain-data/genesis-wasm"
+          {{ .Values.node.command }} export-genesis-wasm  --chain {{ .Values.node.chain }} > /chain-data/genesis-wasm
+        else
+         echo "Genesis head and wasm are in pod ${HOSTNAME%-*}-0"
+        fi
+    volumeMounts:
+      - mountPath: /chain-data
+        name: chain-data
+  - name: dump-session-keys
+    resources:
+      requests:
+        memory: 8Mi
+        cpu: 0.01
+      limits:
+        memory: 32Mi
+    image: docker.io/paritytech/substrate-session-keys-grabber:a5dd354f-20240716
+    args: ["/keystore"]
+    volumeMounts:
+      - mountPath: /keystore
+        name: chain-keystore

charts/node/examples/local-rococo/secret.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: my-key-aura
+stringData:
+  type: aura
+  scheme: sr25519
+  # This is Alice seed. To generate new seed run: `docker run parity/polkadot key generate`
+  seed: 'bottom drive obey lake curtain smoke basket hold race lonely fit walk'
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: my-node-key
+stringData:
+  # To generate new node key run: `docker run -t parity/polkadot key generate-node-key`
+  # 12D3KooWF4B55vTeXa7g88nUdSHVk6DiRXTrsfQyJcg7TrXcCK8U
+  custom-node-key-0: "aaaabbbbbccccddddeeeeffff111122223333444455556666777788889999aab"
+  # 12D3KooWEEZbB6hYKKhMo4hVexcPuTTFGJXbV8q3mobnAC91r8BR
+  custom-node-key-1: "aaaabbbbbccccddddeeeeffff111122223333444455556666777788889999aac"

charts/node/templates/statefulset.yaml

@@ -367,7 +367,11 @@ spec:
               --keystore-path /keystore \
               --key-type $(cat /var/run/secrets/{{ $keys }}/type) \
               --scheme $(cat /var/run/secrets/{{ $keys }}/scheme) \
+              {{- if $.Values.node.existingSecrets.extraDerivation }}
+              --suri "$(cat /var/run/secrets/{{ $keys }}/seed){{ $.Values.node.existingSecrets.extraDerivation }}" \
+              {{- else }}
               --suri /var/run/secrets/{{ $keys }}/seed \
+              {{- end }}
               && echo "Inserted key {{ $keys }} into Keystore" \
               || echo "Failed to insert key {{ $keys }} into Keystore."
               {{- end }}
@@ -602,7 +606,7 @@ spec:
                 --node-key $(cat /custom-node-key/custom-node-key-${POD_INDEX}) \
                 {{- end }}
                 {{- else if .Values.node.existingSecrets.nodeKey }}
-                --node-key $(cat /custom-node-key/{{ .Values.node.existingSecrets.nodeKey.secretKey }}) \
+                --node-key $(cat /custom-node-key/{{ .Values.node.existingSecrets.nodeKey.secretKey }}{{ if .Values.node.existingSecrets.nodeKey.appendPodIndex }}-${POD_INDEX}{{ end }}) \
                 {{- else if .Values.node.vault.nodeKey }}
                 --node-key $(cat /vault/secrets/{{ .Values.node.vault.nodeKey.name }}{{ if .Values.node.vault.nodeKey.vaultKeyAppendPodIndex }}-${POD_INDEX}{{ end }}) \
                 {{- end }}

charts/node/values.yaml

@@ -377,14 +377,21 @@ node:
   existingSecrets:
     # -- List of kubernetes secret names to be added to the keystore.
     # Each secret should contain 3 keys: type, scheme and seed
+    # Secret example: templates/keys.yaml
     # Supercedes node.vault.keys
     keys: []
+    # Add a derivation suffix for the private key.
+    extraDerivation: ""
 
     # -- K8s secret with node key
+    # Secret example: templates/customNodeKeySecret.yaml
     # Supercedes node.vault.nodeKey
     nodeKey: {}
       # secretName: existing-node-secret
       # secretKey: my-node-key
+      # # Append pod index to secret key (e.g., my-node-key -> my-node-key-0)
+      # # Set `appendPodIndex` to true if you want to enable appending the pod index
+      # appendPodIndex: false
 
   # -- Component to inject secrets via annotation of Hashicorp Vault
   # ref: https://www.vaultproject.io/docs/platform/k8s/injector/annotations