From e18f03d4427941f220c4581f40deac2f8cb4b285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AC=9D=E6=98=8E=E8=8F=AF?= Date: Fri, 9 Aug 2024 14:27:18 +0800 Subject: [PATCH] feat(kubernetes): add new Besu Kubernetes chart files --- .../kubernetes/charts/besu-genesis/Chart.yaml | 20 ++ .../besu-genesis/templates/_helpers.tpl | 29 ++ .../templates/genesis-job-cleanup.yaml | 63 ++++ .../templates/genesis-job-init.yaml | 167 +++++++++ .../templates/genesis-service-account.yaml | 45 +++ .../charts/besu-genesis/values.yaml | 50 +++ .../kubernetes/charts/besu-node/Chart.yaml | 20 ++ .../charts/besu-node/templates/_helpers.tpl | 29 ++ .../templates/aws-secret-provider-class.yaml | 38 ++ .../azure-secret-provider-class.yaml | 51 +++ .../templates/besu-config-configmap.yaml | 110 ++++++ .../templates/node-hooks-pre-delete.yaml | 167 +++++++++ .../templates/node-hooks-pre-install.yaml | 197 +++++++++++ .../templates/node-hooks-service-account.yaml | 55 +++ .../templates/node-service-account.yaml | 44 +++ .../besu-node/templates/node-service.yaml | 96 +++++ .../templates/node-servicemonitor.yaml | 0 .../besu-node/templates/node-statefulset.yaml | 333 ++++++++++++++++++ .../besu-node/templates/node-storage.yaml | 70 ++++ .../templates/permissions-configmap.yaml | 20 ++ .../kubernetes/charts/besu-node/values.yaml | 158 +++++++++ 21 files changed, 1762 insertions(+) create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/Chart.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/_helpers.tpl create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-cleanup.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-init.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-service-account.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-genesis/values.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/Chart.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/_helpers.tpl create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/aws-secret-provider-class.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/azure-secret-provider-class.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/besu-config-configmap.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-delete.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-install.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-service-account.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service-account.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-servicemonitor.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-statefulset.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-storage.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/templates/permissions-configmap.yaml create mode 100644 src/eth/instance/infra/kubernetes/charts/besu-node/values.yaml diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/Chart.yaml b/src/eth/instance/infra/kubernetes/charts/besu-genesis/Chart.yaml new file mode 100644 index 0000000..ea93602 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +version: 0.1.0 +appVersion: latest +name: besu-genesis +description: Besu Genesis generator with Helm chart in Kubernetes +keywords: + - ethereum + - besu + - hyperledger + - enterprise + - blockchain + - pegasys + - consensys +home: https://besu.hyperledger.org +sources: + - https://github.com/hyperledger/besu +maintainers: + - name: Joshua Fernandes + email: joshua.fernandes@consensys.net +icon: http://besu.hyperledger.org/en/latest/favicon.ico diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/_helpers.tpl b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/_helpers.tpl new file mode 100644 index 0000000..48be575 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/_helpers.tpl @@ -0,0 +1,29 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "besu-genesis.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "besu-genesis.fullname" -}} +{{- $name := default .Chart.Name -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" $name .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "besu-genesis.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-cleanup.yaml b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-cleanup.yaml new file mode 100644 index 0000000..73ced5c --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-cleanup.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "besu-genesis.name" . }}-cleanup + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: besu-genesis-job-cleanup + app.kubernetes.io/component: genesis-job-cleanup + app.kubernetes.io/part-of: {{ include "besu-genesis.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/managed-by: helm + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook-weight: "0" + helm.sh/hook: "pre-delete" + helm.sh/hook-delete-policy: "hook-succeeded" +spec: + backoffLimit: 3 + completions: 1 + template: + metadata: + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end}} + app.kubernetes.io/name: besu-genesis-job-cleanup + app.kubernetes.io/component: genesis-job-cleanup + app.kubernetes.io/part-of: {{ include "besu-genesis.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/managed-by: helm + spec: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.aws.serviceAccountName }} +{{- else }} + serviceAccountName: {{ include "besu-genesis.name" . }}-sa +{{- end }} + restartPolicy: "Never" + containers: + - name: delete-genesis + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + runAsUser: 0 + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - -c + args: + - | + +{{- if .Values.quorumFlags.removeGenesisOnDelete }} + + echo "Deleting genesis configmap in k8s ..." + kubectl delete configmap --namespace {{ .Release.Namespace }} besu-genesis + + echo "Deleting node-enodes configmap in k8s ..." + kubectl delete configmap --namespace {{ .Release.Namespace }} besu-peers + +{{- end}} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-init.yaml b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-init.yaml new file mode 100644 index 0000000..fd51998 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-job-init.yaml @@ -0,0 +1,167 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "besu-genesis.name" . }}-init + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: besu-genesis-job + app.kubernetes.io/component: genesis-job + app.kubernetes.io/part-of: {{ include "besu-genesis.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook-delete-policy: "hook-succeeded" +spec: + # backoffLimit: 3 + completions: 1 + template: + metadata: + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: besu-genesis-job + app.kubernetes.io/component: genesis-job + app.kubernetes.io/part-of: {{ include "besu-genesis.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/managed-by: helm + spec: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.aws.serviceAccountName }} +{{- else }} + serviceAccountName: {{ include "besu-genesis.name" . }}-sa +{{- end }} + restartPolicy: "Never" + containers: + - name: generate-genesis + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + securityContext: + runAsUser: 0 + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: generated-config + mountPath: /generated-config + command: + - /bin/bash + - -c + args: + - | +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + function safeWriteSecret { + key=$1 + fpath=$2 + az keyvault secret show --vault-name {{ .Values.azure.keyvaultName }} --name $key > /dev/null 2>&1 + if [ $? -ne 0 ]; then + az keyvault secret set --vault-name {{ .Values.azure.keyvaultName }} --name $key --file $fpath --encoding utf-8 + else + # if the key exists pull it from keyvault so that when you update the enodes configmap, you have the right value + az keyvault secret show --vault-name {{ .Values.azure.keyvaultName }} --name $key | jq -r '.value' > $fpath + fi + } + az login --federated-token "$(cat $AZURE_FEDERATED_TOKEN_FILE)" --service-principal -u {{ .Values.azure.identityClientId }} -t {{ .Values.azure.tenantId }} + az account set --subscription {{ .Values.azure.subscriptionId }} + +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + + function safeWriteSecret { + key=$1 + fpath=$2 + aws secretsmanager describe-secret --secret-id $key > /dev/null 2>&1 + if [ $? -ne 0 ]; then + aws secretsmanager create-secret --name $key --description $key --secret-string file://$fpath + else + # if the key exists pull it from keyvault so that when you update the enodes configmap, you have the right value + aws secretsmanager get-secret-value --secret-id $key | jq -r '.SecretString' > $fpath + fi + } + +{{- else }} + + function safeWriteSecret { + key=$1 + fpath=$2 + kubectl create secret generic ${key}-keys --namespace {{ .Release.Namespace }} --from-file=nodekey=${fpath}/nodekey --from-file=nodekey.pub=${fpath}/nodekey.pub --from-file=enode=${fpath}/nodekey.pub --from-file=accountPrivate.key=${fpath}/accountPrivateKey --from-file=accountPassword=${fpath}/accountPassword --from-file=accountKeystore=${fpath}/accountKeystore --from-file=accountAdddress=${fpath}/accountAddress + } + +{{- end }} + + function safeWriteBesuPeersConfigmap { + FOLDER_PATH=$1 + kubectl get configmap --namespace {{ .Release.Namespace }} besu-peers + if [ $? -ne 0 ]; then + kubectl create configmap --namespace {{ .Release.Namespace }} besu-peers --from-file=static-nodes.json=$FOLDER_PATH/static-nodes.json + fi + } + + function safeWriteGenesisConfigmap { + FOLDER_PATH=$1 + kubectl get configmap --namespace {{ .Release.Namespace }} besu-genesis + if [ $? -ne 0 ]; then + kubectl create configmap --namespace {{ .Release.Namespace }} besu-genesis --from-file=genesis.json=$FOLDER_PATH/besu/genesis.json + fi + } + + echo "Creating config ..." + FOLDER_PATH=$(quorum-genesis-tool --consensus {{ .Values.rawGenesisConfig.genesis.config.algorithm.consensus }} {{ if .Values.rawGenesisConfig.blockchain.nodes.generate }} --validators {{ .Values.rawGenesisConfig.blockchain.nodes.count }} {{ else }} --validators 0 {{ end }} --members 0 --bootnodes 0 --chainID {{ .Values.rawGenesisConfig.genesis.config.chainId }} --blockperiod {{ .Values.rawGenesisConfig.genesis.config.algorithm.blockperiodseconds }} --epochLength {{ .Values.rawGenesisConfig.genesis.config.algorithm.epochlength }} --requestTimeout {{ .Values.rawGenesisConfig.genesis.config.algorithm.requesttimeoutseconds }} --difficulty {{ .Values.rawGenesisConfig.genesis.difficulty }} --gasLimit {{ .Values.rawGenesisConfig.genesis.gasLimit }} --coinbase {{ .Values.rawGenesisConfig.genesis.coinbase }} {{ if .Values.rawGenesisConfig.blockchain.accountPassword }} --accountPassword {{ .Values.rawGenesisConfig.blockchain.accountPassword }} {{ end }} --quickstartDevAccounts {{ .Values.rawGenesisConfig.genesis.includeQuickStartAccounts }} --alloc {{ if .Values.rawGenesisConfig.genesis.alloc }} '{{ .Values.rawGenesisConfig.genesis.alloc | toJson }}' {{ else }} '{}' {{ end }} --outputPath /generated-config | tail -1 | sed -e "s/^Artifacts in folder: //") + + echo "Creating bootnodes configmap in k8s ..." + echo "[]" > /tmp/besu-bootnodes + kubectl create configmap --namespace {{ .Release.Namespace }} besu-bootnodes --from-file=bootnodes=/tmp/besu-bootnodes + + echo $FOLDER_PATH + echo "Creating genesis configmap in k8s ..." + safeWriteGenesisConfigmap $FOLDER_PATH + + # create the static-nodes with proper dns names for the quorum nodes + echo "[" > $FOLDER_PATH/static-nodes.json + echo "Creating validator secrets in k8s ..." + i=1 + for f in $(find $FOLDER_PATH -type d -iname "validator*" -exec basename {} \;); do + if [ -d $FOLDER_PATH/${f} ]; then + echo $f + echo "Creating keys in vault for validator-${i} ..." + +{{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + + safeWriteSecret besu-node-validator-${i}-nodekey $FOLDER_PATH/${f}/nodekey + safeWriteSecret besu-node-validator-${i}-nodekeypub $FOLDER_PATH/${f}/nodekey.pub + safeWriteSecret besu-node-validator-${i}-enode $FOLDER_PATH/${f}/nodekey.pub + safeWriteSecret besu-node-validator-${i}-address $FOLDER_PATH/${f}/address + kubectl create configmap --namespace {{ .Release.Namespace }} besu-node-validator-${i}-address --from-file=address=$FOLDER_PATH/${f}/address + + safeWriteSecret besu-node-validator-${i}-accountPrivateKey $FOLDER_PATH/${f}/accountPrivateKey + safeWriteSecret besu-node-validator-${i}-accountPassword $FOLDER_PATH/${f}/accountPassword + safeWriteSecret besu-node-validator-${i}-accountKeystore $FOLDER_PATH/${f}/accountKeystore + safeWriteSecret besu-node-validator-${i}-accountAddress $FOLDER_PATH/${f}/accountAddress + +{{- else }} + + safeWriteSecret besu-node-validator-${i} "$FOLDER_PATH/${f}" + kubectl create configmap --namespace {{ .Release.Namespace }} besu-node-validator-${i}-address --from-file=address=$FOLDER_PATH/${f}/address + +{{- end }} + # add to the static-nodes + pubkey=$(cat $FOLDER_PATH/${f}/nodekey.pub ) + echo ",\"enode://$pubkey@besu-node-validator-$i-0.besu-node-validator-$i.{{ .Release.Namespace }}.svc.cluster.local:30303?discport=0\"" >> $FOLDER_PATH/static-nodes.json + + i=$((i+1)) + fi + done + + echo "]" >> $FOLDER_PATH/static-nodes.json + # remove the extra comma to make it valid json + sed -i '0,/,/s///' $FOLDER_PATH/static-nodes.json + safeWriteBesuPeersConfigmap $FOLDER_PATH + + echo "Completed ..." + + volumes: + - name: generated-config + emptyDir: {} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-service-account.yaml b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-service-account.yaml new file mode 100644 index 0000000..666fabb --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/templates/genesis-service-account.yaml @@ -0,0 +1,45 @@ + +{{- if not .Values.cluster.cloudNativeServices }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "besu-genesis.name" . }}-sa + namespace: {{ .Release.Namespace }} + +{{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "besu-genesis.name" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: ["create", "get", "list", "update", "delete" ] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "besu-genesis.name" . }}-rb + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "besu-genesis.name" . }}-role +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.aws.serviceAccountName }} +{{- else }} + name: {{ include "besu-genesis.name" . }}-sa +{{- end}} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-genesis/values.yaml b/src/eth/instance/infra/kubernetes/charts/besu-genesis/values.yaml new file mode 100644 index 0000000..b35ac4a --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-genesis/values.yaml @@ -0,0 +1,50 @@ +--- + +quorumFlags: + removeGenesisOnDelete: true + +cluster: + provider: local # choose from: local | aws | azure + cloudNativeServices: false # set to true to use Cloud Native Services (SecretsManager and IAM for AWS; KeyVault & Managed Identities for Azure) + +aws: + # the aws cli commands uses the name 'quorum-sa' so only change this if you altered the name + serviceAccountName: quorum-sa + # the region you are deploying to + region: ap-southeast-2 + +azure: + serviceAccountName: quorum-sa + # the clientId of the user assigned managed identity created in the template + identityClientId: azure-clientId + keyvaultName: azure-keyvault + # the tenant ID of the key vault + tenantId: azure-tenantId + # the subscription ID to use - this needs to be set explicitly when using multi tenancy + subscriptionId: azure-subscriptionId + +# the raw Genesis config +# rawGenesisConfig.blockchain.nodes set the number of validators/signers +rawGenesisConfig: + genesis: + config: + chainId: 1337 + algorithm: + consensus: qbft + blockperiodseconds: 10 + epochlength: 30000 + requesttimeoutseconds: 20 + gasLimit: '0xf7b760' + difficulty: '0x1' + coinbase: '0x0000000000000000000000000000000000000000' + blockchain: + nodes: + generate: true + count: 4 + accountPassword: 'password' + + +image: + repository: consensys/quorum-k8s-hooks + tag: qgt-0.2.15 + pullPolicy: IfNotPresent \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/Chart.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/Chart.yaml new file mode 100644 index 0000000..5b0e1b8 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +version: 0.1.0 +appVersion: 22.1.3 +name: besu-node +description: Besu nodes for a POA network +keywords: + - ethereum + - besu + - hyperledger + - enterprise + - blockchain + - pegasys + - consensys +home: https://besu.hyperledger.org +sources: + - https://github.com/hyperledger/besu +maintainers: + - name: Joshua Fernandes + email: joshua.fernandes@consensys.net +icon: http://besu.hyperledger.org/en/latest/favicon.ico diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/_helpers.tpl b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/_helpers.tpl new file mode 100644 index 0000000..ff3333d --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/_helpers.tpl @@ -0,0 +1,29 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "besu-node.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "besu-node.fullname" -}} +{{- $name := default .Chart.Name -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" $name .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "besu-node.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/aws-secret-provider-class.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/aws-secret-provider-class.yaml new file mode 100644 index 0000000..022065f --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/aws-secret-provider-class.yaml @@ -0,0 +1,38 @@ +{{- if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: {{ include "besu-node.fullname" . }}-secret-provider + namespace: {{ .Release.Namespace }} +spec: + provider: aws + parameters: + objects: | + - objectName: {{ include "besu-node.fullname" . }}-nodekey + objectAlias: nodekey + objectType: secretsmanager + objectVersion: "" + - objectName: {{ include "besu-node.fullname" . }}-nodekeypub + objectAlias: nodekey.pub + objectType: secretsmanager + objectVersion: "" + - objectName: {{ include "besu-node.fullname" . }}-enode + objectAlias: enode + objectType: secretsmanager + objectVersion: "" + {{- if .Values.quorumFlags.privacy }} + - objectName: {{ include "besu-node.fullname" . }}-tmkey + objectAlias: tm.key + objectType: secretsmanager + objectVersion: "" + - objectName: {{ include "besu-node.fullname" . }}-tmkeypub + objectAlias: tm.pub + objectType: secretsmanager + objectVersion: "" + - objectName: {{ include "besu-node.fullname" . }}-tmpassword + objectAlias: tm.password + objectType: secretsmanager + objectVersion: "" + {{- end }} +{{- end }} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/azure-secret-provider-class.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/azure-secret-provider-class.yaml new file mode 100644 index 0000000..b9654f8 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/azure-secret-provider-class.yaml @@ -0,0 +1,51 @@ +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ include "besu-node.fullname" . }}-secret-provider + namespace: {{ .Release.Namespace }} +spec: + provider: azure + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "true" + userAssignedIdentityID: "{{ .Values.azure.nodePoolIdentityClientId }}" + keyvaultName: "{{ .Values.azure.keyvaultName }}" + tenantId: "{{ .Values.azure.tenantId }}" + cloudName: "AzurePublicCloud" + objects: | + array: + - | + objectName: {{ include "besu-node.fullname" . }}-nodekey + objectAlias: nodekey + objectType: secret + objectVersion: "" + - | + objectName: {{ include "besu-node.fullname" . }}-nodekeypub + objectAlias: nodekey.pub + objectType: secret + objectVersion: "" + - | + objectName: {{ include "besu-node.fullname" . }}-enode + objectAlias: enode + objectType: secret + objectVersion: "" + {{- if .Values.quorumFlags.privacy }} + - | + objectName: {{ include "besu-node.fullname" . }}-tmkey + objectAlias: tm.key + objectType: secret + objectVersion: "" + - | + objectName: {{ include "besu-node.fullname" . }}-tmkeypub + objectAlias: tm.pub + objectType: secret + objectVersion: "" + - | + objectName: {{ include "besu-node.fullname" . }}-tmpassword + objectAlias: tm.password + objectType: secret + objectVersion: "" + {{- end }} +{{- end }} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/besu-config-configmap.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/besu-config-configmap.yaml new file mode 100644 index 0000000..eb77ade --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/besu-config-configmap.yaml @@ -0,0 +1,110 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "besu-node.fullname" . }}-besu-config + labels: + app.kubernetes.io/name: besu-config + app.kubernetes.io/component: besu + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + namespace: {{ .Release.Namespace }} +data: + config.toml: |- + # Every possible CLI should be in this file. + # + # Please use a plausible value, besu has to at least be able to parse it. + # If it is a multi-valued CLI make it a TOML array. + # If it is a number or boolean make it a number or boolean + # All other config options are strings, and must be quoted. + + # Node Information + data-path={{ .Values.node.besu.dataPath | quote }} + genesis-file={{ .Values.node.besu.genesisFilePath | quote }} + logging={{ .Values.node.besu.logging | quote }} + node-private-key-file={{.Values.node.besu.privateKeyPath | quote }} + + {{ if .Values.node.besu.p2p.enabled -}} + # P2P network + p2p-enabled={{ .Values.node.besu.p2p.enabled }} + discovery-enabled={{ .Values.node.besu.p2p.discovery }} + static-nodes-file={{ .Values.node.besu.p2p.staticNodes | quote }} + p2p-host={{ .Values.node.besu.p2p.host | quote }} + p2p-port={{ .Values.node.besu.p2p.port }} + max-peers={{ .Values.node.besu.p2p.maxPeers }} + {{ end }} + + {{ if or .Values.node.besu.rpc.enabled .Values.node.besu.graphql.enabled .Values.node.besu.ws.enabled }} + host-allowlist={{ .Values.node.besu.http.allowlist }} + {{ end }} + + {{ if .Values.node.besu.rpc.enabled -}} + # JSON-RPC + rpc-http-enabled={{ .Values.node.besu.rpc.enabled }} + rpc-http-host={{ .Values.node.besu.rpc.host | quote }} + rpc-http-port={{ .Values.node.besu.rpc.port }} + rpc-http-api={{ .Values.node.besu.rpc.api }} + rpc-http-cors-origins={{ .Values.node.besu.rpc.corsOrigins }} + rpc-http-authentication-enabled={{ .Values.node.besu.rpc.authenticationEnabled }} + rpc-http-max-active-connections={{ .Values.node.besu.rpc.maxActiveConnections }} + revert-reason-enabled=true + {{ end }} + + {{ if .Values.node.besu.graphql.enabled -}} + # GRAPHQL-RPC + graphql-http-enabled={{ .Values.node.besu.graphql.enabled }} + graphql-http-host={{ .Values.node.besu.graphql.host | quote }} + graphql-http-port={{ .Values.node.besu.graphql.port }} + graphql-http-cors-origins={{ .Values.node.besu.graphql.corsOrigins }} + {{ end }} + + {{ if .Values.node.besu.ws.enabled -}} + # WebSockets API + rpc-ws-enabled={{ .Values.node.besu.ws.enabled }} + rpc-ws-host={{ .Values.node.besu.ws.host | quote }} + rpc-ws-port={{ .Values.node.besu.ws.port }} + rpc-ws-api={{ .Values.node.besu.ws.api }} + rpc-ws-authentication-enabled={{ .Values.node.besu.ws.authenticationEnabled }} + {{ end }} + + {{ if .Values.node.besu.permissions.enabled -}} + # Permissioning + permissions-nodes-config-file-enabled={{ .Values.node.besu.permissions.nodes.enabled }} + permissions-nodes-config-file={{ .Values.node.besu.permissions.filePath | quote }} + permissions-accounts-config-file-enabled={{ .Values.node.besu.permissions.accounts.enabled }} + permissions-accounts-config-file={{ .Values.node.besu.permissions.filePath | quote }} + permissions-nodes-contract-enabled={{ .Values.node.besu.permissions.nodesContract.enabled }} + permissions-nodes-contract-address={{ .Values.node.besu.permissions.nodesContract.address | quote }} + permissions-accounts-contract-enabled={{ .Values.node.besu.permissions.accountsContract.enabled }} + permissions-accounts-contract-address={{ .Values.node.besu.permissions.accountsContract.address | quote }} + {{ end }} + + {{ if .Values.quorumFlags.privacy -}} + # Privacy + privacy-enabled={{ .Values.quorumFlags.privacy }} + privacy-url={{ .Values.node.besu.privacy.url | quote }} + {{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + privacy-public-key-file="{{ .Values.node.besu.keysPath }}/{{ .Values.node.besu.privacy.pubkeyFile }}" + {{ else }} + privacy-public-key-file="{{ .Values.node.besu.privacy.pubkeysPath }}/{{ .Values.node.besu.privacy.pubkeyFile }}" + {{ end }} + privacy-onchain-groups-enabled={{ .Values.node.besu.privacy.onchainGroupsEnabled }} + {{ end }} + + {{ if .Values.node.besu.metrics.enabled -}} + # Metrics + metrics-enabled={{ .Values.node.besu.metrics.enabled }} + metrics-host={{ .Values.node.besu.metrics.host | quote }} + metrics-port={{ .Values.node.besu.metrics.port }} + {{ end }} + + {{ if .Values.node.besu.metricsPush.enabled -}} + # Metrics Push + metrics-push-enabled={{ .Values.node.besu.metricsPush.enabled }} + metrics-push-host={{ .Values.node.besu.metricsPush.host | quote }} + metrics-push-port={{ .Values.node.besu.metricsPush.port }} + metrics-push-interval={{ .Values.node.besu.metricsPush.interval }} + metrics-push-prometheus-job={{ .Values.node.besu.metricsPush.prometheusJob | quote }} + {{ end }} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-delete.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-delete.yaml new file mode 100644 index 0000000..4855d46 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-delete.yaml @@ -0,0 +1,167 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "besu-node.fullname" . }}-pre-delete-hook + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: "hook-succeeded" + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: pre-delete-hook + app.kubernetes.io/component: job + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm +spec: + backoffLimit: 3 + completions: 1 + template: + metadata: + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end}} + app.kubernetes.io/name: pre-delete-hook + app.kubernetes.io/instance: {{ .Release.Name }} + spec: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.aws.serviceAccountName }} +{{- else }} + serviceAccountName: {{ include "besu-node.fullname" . }}-hooks-sa +{{- end }} + restartPolicy: "OnFailure" + containers: + - name: {{ template "besu-node.fullname" . }}-node-pre-delete-hook + image: "{{ .Values.image.hooks.repository }}:{{ .Values.image.hooks.tag }}" + imagePullPolicy: {{ .Values.image.hooks.pullPolicy }} + command: + - /bin/bash + - -c + args: + - | + + echo "{{ template "besu-node.fullname" . }} Pre Delete hook ..." + +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + + function deleteSecret { + key=$1 + fpath=$2 + az keyvault secret show --vault-name {{ .Values.azure.keyvaultName }} --name $key > /dev/null 2>&1 + if [ $? -eq 0 ]; then + az keyvault secret delete --vault-name {{ .Values.azure.keyvaultName }} --name $key + fi + } + + az login --federated-token "$(cat $AZURE_FEDERATED_TOKEN_FILE)" --service-principal -u {{ .Values.azure.identityClientId }} -t {{ .Values.azure.tenantId }} + az account set --subscription {{ .Values.azure.subscriptionId }} + +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + + function deleteSecret { + key=$1 + aws secretsmanager describe-secret --secret-id $key > /dev/null 2>&1 + if [ $? -eq 0 ]; then + aws secretsmanager delete-secret --secret-id $key --recovery-window-in-days 7 + fi + } + +{{- else }} + + function deleteSecret { + key=$1 + kubectl delete secret ${key} --namespace {{ .Release.Namespace }} + } + +{{- end }} + + function delete_node_from_tessera_peers_configmap { + kubectl -n {{ .Release.Namespace }} get configmap tessera-peers -o json + # if there is no configmap, do nothing + if [ $? -ne 0 ]; then + echo "No tessera-peers found, nothing to do..." + # delete the one + else + echo "tessera-peers found, deleting {{ template "besu-node.fullname" . }}..." + echo $(kubectl -n {{ .Release.Namespace }} get configmap tessera-peers -o jsonpath='{.data.tesseraPeers}' ) > /tmp/tessera-peers.raw + cat /tmp/tessera-peers.raw | jq --arg NEEDLE "{{ template "besu-node.fullname" . }}" 'del(.[] | select( .url | contains($NEEDLE) ))' > /tmp/tessera-peers + kubectl -n {{ .Release.Namespace }} create configmap tessera-peers --from-file=tesseraPeers=/tmp/tessera-peers -o yaml --dry-run=client | kubectl replace -f - + fi + } + + function delete_node_from_enodes_configmap { + kubectl -n {{ .Release.Namespace }} get configmap besu-peers -o json + # if there is no configmap, do nothing + if [ $? -ne 0 ]; then + echo "No peers found, nothing to do..." + # delete the one + else + echo "besu-peers found, deleting {{ template "besu-node.fullname" . }}..." + echo $(kubectl -n {{ .Release.Namespace }} get configmap besu-peers -o jsonpath='{.data.static-nodes\.json}' ) > /tmp/static-nodes.json.raw + cat /tmp/static-nodes.json.raw | jq --arg NEEDLE "{{ template "besu-node.fullname" . }}" 'del(.[] | select( . | contains($NEEDLE) ))' > /tmp/static-nodes.json + kubectl -n {{ .Release.Namespace }} create configmap besu-peers --from-file=static-nodes.json=/tmp/static-nodes.json -o yaml --dry-run=client | kubectl replace -f - + + echo "Deleting node address configmap... " + kubectl delete configmap {{ template "besu-node.fullname" . }}-address --namespace {{ .Release.Namespace }} + fi + } + + + function delete_node_from_besu_bootnodes_configmap { + kubectl -n {{ .Release.Namespace }} get configmap besu-bootnodes -o json + # if there is no configmap, do nothing + if [ $? -ne 0 ]; then + echo "No bootnodes found, nothing to do..." + # delete the one + else + echo "besu-bootnodes found, deleting {{ template "besu-node.fullname" . }}..." + echo $(kubectl -n {{ .Release.Namespace }} get configmap besu-bootnodes -o jsonpath='{.data.bootnodes-json}' ) > /tmp/besu-bootnodes-json.raw + cat /tmp/besu-bootnodes-json.raw | jq --arg NEEDLE "{{ template "besu-node.fullname" . }}" 'del(.[] | select( . | contains($NEEDLE) ))' > /tmp/besu-bootnodes-json + cat /tmp/besu-bootnodes-json | jq -r -c '. | join(",")' | tr -d '\n' > /tmp/besu-bootnodes-string + kubectl -n {{ .Release.Namespace }} create configmap besu-bootnodes --from-file=bootnodes-json=/tmp/besu-bootnodes-json --from-file=bootnodes-string=/tmp/besu-bootnodes-string -o yaml --dry-run=client | kubectl replace -f - + fi + } + + delete_node_from_enodes_configmap + delete_node_from_besu_bootnodes_configmap + delete_node_from_tessera_peers_configmap + +{{- if .Values.quorumFlags.removeKeysOnDelete }} + +{{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + + deleteSecret {{ template "besu-node.fullname" . }}-nodekey + deleteSecret {{ template "besu-node.fullname" . }}-nodekeypub + deleteSecret {{ template "besu-node.fullname" . }}-enode + deleteSecret {{ template "besu-node.fullname" . }}-accountPrivateKey + deleteSecret {{ template "besu-node.fullname" . }}-accountPassword + deleteSecret {{ template "besu-node.fullname" . }}-accountKeystore + deleteSecret {{ template "besu-node.fullname" . }}-accountAddress + deleteSecret {{ template "besu-node.fullname" . }}-address + +{{- if .Values.quorumFlags.privacy }} + deleteSecret {{ template "besu-node.fullname" . }}-tmkey + deleteSecret {{ template "besu-node.fullname" . }}-tmkeypub + deleteSecret {{ template "besu-node.fullname" . }}-tmpassword +{{- end }} + +{{- else }} + deleteSecret {{ template "besu-node.fullname" . }}-keys + deleteSecret {{ template "besu-node.fullname" . }}-account +{{- if .Values.quorumFlags.privacy }} + deleteSecret {{ template "besu-node.fullname" . }}-tessera-keys +{{- end }} + +{{- end }} + +{{- end }} + + echo "Completed" \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-install.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-install.yaml new file mode 100644 index 0000000..a79f95a --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-pre-install.yaml @@ -0,0 +1,197 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "besu-node.fullname" . }}-pre-install-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": "hook-succeeded" + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: pre-install-hook + app.kubernetes.io/component: job + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm +spec: + backoffLimit: 1 + completions: 1 + template: + metadata: + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: pre-install-hook + app.kubernetes.io/instance: {{ .Release.Name }} + spec: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.aws.serviceAccountName }} +{{- else }} + serviceAccountName: {{ include "besu-node.fullname" . }}-hooks-sa +{{- end }} + restartPolicy: "OnFailure" + containers: + - name: {{ template "besu-node.fullname" . }}-pre-start-hook + image: {{ .Values.image.hooks.repository }}:{{ .Values.image.hooks.tag }} + imagePullPolicy: {{ .Values.image.besu.pullPolicy }} + securityContext: + runAsUser: 0 + command: + - /bin/bash + - -c + args: + - | + + function update_peers_configmap { + PUBKEY_LOC=$1 + kubectl -n {{ .Release.Namespace }} get configmap besu-peers -o json + if [ $? -ne 0 ]; then + echo "[]" > /tmp/static-nodes.json.raw + else + echo $(kubectl -n {{ .Release.Namespace }} get configmap besu-peers -o jsonpath='{.data.static-nodes\.json}' ) > /tmp/static-nodes.json.raw + fi + # update the entries + echo "updating besu-peers..." + pubkey=$(cat $PUBKEY_LOC ) + NEEDLE="enode://$pubkey@{{ template "besu-node.fullname" . }}-0.{{ template "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:30303?discport=0" + cat /tmp/static-nodes.json.raw | jq --arg NEEDLE "$NEEDLE" '. += [ $NEEDLE ] | unique ' > /tmp/static-nodes.json + kubectl -n {{ .Release.Namespace }} create configmap besu-peers --from-file=static-nodes.json=/tmp/static-nodes.json -o yaml --dry-run=client | kubectl replace -f - + } + + function update_tessera_peers_configmap { + kubectl -n {{ .Release.Namespace }} get configmap tessera-peers -o json + # first time a tx node is deployed and there is no configmap + if [ $? -ne 0 ]; then + echo "No tessera-peers found, creating a new one..." + echo "[{ \"url\": \"http://{{ template "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:9000\" }]" > /tmp/tessera-peers + kubectl --namespace {{ .Release.Namespace }} create configmap tessera-peers --from-file=tesseraPeers=/tmp/tessera-peers + + # update the entries + else + echo "Tessera-peers found, updating existing..." + echo $(kubectl -n {{ .Release.Namespace }} get configmap tessera-peers -o jsonpath='{.data.tesseraPeers}' ) > /tmp/tessera-peers.raw + NEEDLE="http://{{ template "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:9000" + cat /tmp/tessera-peers.raw | jq --arg NEEDLE "$NEEDLE" '. += [{"url": $NEEDLE}] | unique ' > /tmp/tessera-peers + kubectl -n {{ .Release.Namespace }} create configmap tessera-peers --from-file=tesseraPeers=/tmp/tessera-peers -o yaml --dry-run=client | kubectl replace -f - + fi + } + + function update_bootnodes_configmap { + PUBKEY_LOC=$1 + kubectl -n {{ .Release.Namespace }} get configmap besu-bootnodes -o json + if [ $? -ne 0 ]; then + echo "[]" > /tmp/besu-bootnodes-json.raw + kubectl -n {{ .Release.Namespace }} create configmap besu-bootnodes --from-file=bootnodes-json=/tmp/besu-bootnodes-json.raw --from-literal=bootnodes-string="" + fi + + echo "updating besu-bootnodes..." + echo $(kubectl -n {{ .Release.Namespace }} get configmap besu-bootnodes -o jsonpath='{.data.bootnodes-json}' ) > /tmp/besu-bootnodes-json.raw + pubkey=$(cat $PUBKEY_LOC ) + NEEDLE="enode://$pubkey@{{ template "besu-node.fullname" . }}-0.{{ template "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:30303" + cat /tmp/besu-bootnodes-json.raw | jq --arg NEEDLE "$NEEDLE" '. += [ $NEEDLE ] | unique ' > /tmp/besu-bootnodes-json + cat /tmp/besu-bootnodes-json | jq -r -c '. | join(",")' | tr -d '\n' > /tmp/besu-bootnodes-string + kubectl -n {{ .Release.Namespace }} create configmap besu-bootnodes --from-file=bootnodes-json=/tmp/besu-bootnodes-json --from-file=bootnodes-string=/tmp/besu-bootnodes-string -o yaml --dry-run=client | kubectl replace -f - + } + + +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + function safeWriteSecret { + key=$1 + fpath=$2 + az keyvault secret show --vault-name {{ .Values.azure.keyvaultName }} --name $key > /dev/null 2>&1 + if [ $? -ne 0 ]; then + az keyvault secret set --vault-name {{ .Values.azure.keyvaultName }} --name $key --file $fpath --encoding utf-8 + else + # if the key exists pull it from keyvault so that when you update the enodes configmap, you have the right value + az keyvault secret show --vault-name {{ .Values.azure.keyvaultName }} --name $key | jq -r '.value' > $fpath + fi + } + + az login --federated-token "$(cat $AZURE_FEDERATED_TOKEN_FILE)" --service-principal -u $AZURE_CLIENT_ID -t $AZURE_TENANT_ID + az account set --subscription {{ .Values.azure.subscriptionId }} + +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + function safeWriteSecret { + key=$1 + fpath=$2 + aws secretsmanager describe-secret --secret-id $key > /dev/null 2>&1 + if [ $? -ne 0 ]; then + aws secretsmanager create-secret --name $key --description $key --secret-string file://$fpath + else + # if the key exists pull it from keyvault so that when you update the enodes configmap, you have the right value + aws secretsmanager get-secret-value --secret-id $key | jq -r '.SecretString' > $fpath + fi + } +{{- else }} + + function safeWriteSecret { + key=$1 + fpath=$2 + kubectl get secret ${key}-keys --namespace {{ .Release.Namespace }} -o json > /dev/null 2>&1 + if [ $? -ne 0 ]; then + kubectl create secret generic ${key}-keys --namespace {{ .Release.Namespace }} --from-file=nodekey=${fpath}/nodekey --from-file=nodekey.pub=${fpath}/nodekey.pub --from-file=enode=${fpath}/nodekey.pub --from-file=accountPrivate.key=${fpath}/accountPrivateKey --from-file=accountPassword=${fpath}/accountPassword --from-file=accountKeystore=${fpath}/accountKeystore --from-file=accountAdddress=${fpath}/accountAddress + else + # if the key exists pull it from secrets so that when you update the enodes configmap, you have the right value + kubectl get secrets ${key}-keys --namespace {{ .Release.Namespace }} -o json | jq '.data.enode' | tr -d '"'| base64 --decode > /tmp/enode + fi + + kubectl get secret ${key}-tessera-keys --namespace {{ .Release.Namespace }} -o json > /dev/null 2>&1 + if [ $? -ne 0 ]; then + kubectl create secret generic {{ template "besu-node.fullname" . }}-tessera-keys --namespace {{ .Release.Namespace }} --from-file=tm.key=$FOLDER_PATH/member0/tessera.key --from-file=tm.pub=$FOLDER_PATH/member0/tessera.pub --from-file=tm.password=$FOLDER_PATH/member0/passwordFile.txt + fi + } +{{- end }} + + echo "{{ template "besu-node.fullname" . }} hook ..." + echo "Nodekey generation ..." + FOLDER_PATH=$(quorum-genesis-tool --validators 0 --members 1 --bootnodes 0 {{ if .Values.node.besu.account.password }} --accountPassword {{ .Values.node.besu.account.password }} {{ end }} --outputPath /generated-config | tail -1 | sed -e "s/^Artifacts in folder: //") + echo "Creating {{ template "besu-node.fullname" . }} secrets in k8s ..." + +{{- if .Values.cluster.cloudNativeServices }} + echo "Creating keys in vault for {{ template "besu-node.fullname" . }} ..." + safeWriteSecret {{ template "besu-node.fullname" . }}-nodekey $FOLDER_PATH/member0/nodekey + safeWriteSecret {{ template "besu-node.fullname" . }}-nodekeypub $FOLDER_PATH/member0/nodekey.pub + safeWriteSecret {{ template "besu-node.fullname" . }}-enode $FOLDER_PATH/member0/nodekey.pub + safeWriteSecret {{ template "besu-node.fullname" . }}-address $FOLDER_PATH/member0/address + safeWriteSecret {{ template "besu-node.fullname" . }}-accountPrivateKey $FOLDER_PATH/member0/accountPrivateKey + safeWriteSecret {{ template "besu-node.fullname" . }}-accountPassword $FOLDER_PATH/member0/accountPassword + safeWriteSecret {{ template "besu-node.fullname" . }}-accountKeystore $FOLDER_PATH/member0/accountKeystore + safeWriteSecret {{ template "besu-node.fullname" . }}-accountAddress $FOLDER_PATH/member0/accountAddress +{{- else }} + safeWriteSecret {{ template "besu-node.fullname" . }} $FOLDER_PATH/member0 +{{- end }} + cat $FOLDER_PATH/member0/nodekey.pub > /tmp/enode + echo "Creating configmap for node address" + kubectl create configmap {{ template "besu-node.fullname" . }}-address --from-file=address=$FOLDER_PATH/${f}/member0/address + + update_peers_configmap /tmp/enode +{{- if .Values.quorumFlags.isBootnode }} + update_bootnodes_configmap /tmp/enode +{{- end }} + echo "Completed" + +{{- if .Values.quorumFlags.privacy }} + FOLDER_PATH=$(quorum-genesis-tool --validators 0 --members 1 --bootnodes 0 --tesseraEnabled true --tesseraPassword {{ .Values.node.tessera.password }} --outputPath /tmp/tessera | tail -1 | sed -e "s/^Artifacts in folder: //") + if [ ! -f "$FOLDER_PATH/member0/passwordFile.txt" ]; then + echo "" > $FOLDER_PATH/member0/passwordFile.txt + fi + echo "Creating {{ template "besu-node.fullname" . }}-tessera-keys secrets in k8s ..." +{{- if .Values.cluster.cloudNativeServices }} + safeWriteSecret {{ template "besu-node.fullname" . }}-tmkey $FOLDER_PATH/member0/tessera.key + safeWriteSecret {{ template "besu-node.fullname" . }}-tmkeypub $FOLDER_PATH/member0/tessera.pub + safeWriteSecret {{ template "besu-node.fullname" . }}-tmpassword $FOLDER_PATH/member0/passwordFile.txt +{{- else }} + safeWriteSecret {{ template "besu-node.fullname" . }} $FOLDER_PATH/member0 +{{- end }} + update_tessera_peers_configmap + echo "Tessera Completed" +{{- end }} + echo "hook completed" \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-service-account.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-service-account.yaml new file mode 100644 index 0000000..df7026a --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-hooks-service-account.yaml @@ -0,0 +1,55 @@ +{{- if not .Values.cluster.cloudNativeServices }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "besu-node.fullname" . }}-hooks-sa + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook": "pre-install,pre-delete,post-delete" +{{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "besu-node.fullname" . }}-hooks-role + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook": "pre-install,pre-delete,post-delete" +rules: + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: ["create", "get", "list", "update", "delete", "patch" ] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "besu-node.fullname" . }}-hooks-rb + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook": "pre-install,pre-delete,post-delete" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "besu-node.fullname" . }}-hooks-role +subjects: +- kind: ServiceAccount + namespace: {{ .Release.Namespace }} +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.aws.serviceAccountName }} +{{- else }} + name: {{ include "besu-node.fullname" . }}-hooks-sa +{{- end}} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service-account.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service-account.yaml new file mode 100644 index 0000000..475ab7b --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service-account.yaml @@ -0,0 +1,44 @@ + +{{- if not .Values.cluster.cloudNativeServices }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "besu-node.fullname" . }}-sa + namespace: {{ .Release.Namespace }} +{{- end }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "besu-node.fullname" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "besu-node.fullname" . }}-rb + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "besu-node.fullname" . }}-role +subjects: +- kind: ServiceAccount + namespace: {{ .Release.Namespace }} +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + name: {{ .Values.aws.serviceAccountName }} +{{- else }} + name: {{ include "besu-node.fullname" . }}-sa +{{- end}} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service.yaml new file mode 100644 index 0000000..0723eec --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-service.yaml @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "besu-node.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "besu-node.fullname" . }} + app.kubernetes.io/component: service + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + ports: + - name: json-rpc + port: {{ .Values.node.besu.rpc.port }} + targetPort: json-rpc + protocol: TCP + - name: ws + port: {{ .Values.node.besu.ws.port }} + targetPort: ws + protocol: TCP + - name: graphql + port: {{ .Values.node.besu.graphql.port }} + targetPort: graphql + protocol: TCP + - name: rlpx + port: {{ .Values.node.besu.p2p.port }} + targetPort: rlpx + protocol: TCP + - name: discovery + port: {{ .Values.node.besu.p2p.port }} + targetPort: discovery + protocol: UDP + - name: metrics + port: {{ .Values.node.besu.metrics.port }} + targetPort: metrics + protocol: TCP + +{{- if .Values.quorumFlags.privacy }} + - name: tessera + port: {{ .Values.node.tessera.port }} + targetPort: tessera + protocol: TCP + - name: tessera-tp + port: {{ .Values.node.tessera.tpport }} + targetPort: tessera-tp + protocol: TCP + - name: tessera-q2t + port: {{ .Values.node.tessera.q2tport }} + targetPort: tessera-q2t + protocol: TCP +{{- end }} + + +{{- if and .Values.node.besu.metrics.enabled .Values.node.besu.metrics.serviceMonitorEnabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "besu-node.fullname" . }}-servicemonitor + labels: + release: monitoring + helm.sh/chart: {{ include "besu-node.chart" . }} + app: {{ template "besu-node.fullname" . }} + chart: {{ template "besu-node.chart" . }} + heritage: {{ .Release.Service }} + namespace: {{ .Release.Namespace }} + app.kubernetes.io/name: {{ include "besu-node.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/component: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "besu-node.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: {{ .Release.Name }} + endpoints: + - port: metrics + interval: 15s + path: /metrics + scheme: http + honorLabels: true +{{- end }} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-servicemonitor.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-servicemonitor.yaml new file mode 100644 index 0000000..e69de29 diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-statefulset.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-statefulset.yaml new file mode 100644 index 0000000..a4326d2 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-statefulset.yaml @@ -0,0 +1,333 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "besu-node.fullname" . }} + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: besu-statefulset + app.kubernetes.io/component: besu + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + {{- range $labelName, $labelValue := .Values.node.besu.customLabels }} + {{ $labelName }}: {{ $labelValue }} + {{- end }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + podManagementPolicy: OrderedReady + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: besu-statefulset + app.kubernetes.io/component: besu + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + serviceName: {{ include "besu-node.fullname" . }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ include "besu-node.fullname" . }}-storage + resources: + requests: + storage: "{{ .Values.storage.pvcSizeLimit }}" + template: + metadata: + labels: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + azure.workload.identity/use: "true" +{{- end }} + app.kubernetes.io/name: besu-statefulset + app.kubernetes.io/component: besu + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.node.besu.metrics.port | quote}} + prometheus.io/path: "/metrics" + spec: +{{- if and (eq .Values.cluster.provider "azure") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.azure.serviceAccountName }} +{{- else if and (eq .Values.cluster.provider "aws") (.Values.cluster.cloudNativeServices) }} + serviceAccountName: {{ .Values.aws.serviceAccountName }} +{{- else }} + serviceAccountName: {{ include "besu-node.fullname" . }}-sa +{{- end }} + initContainers: + +{{- if has .Values.cluster.provider .Values.volumePermissionsFix }} + # fix for minikube and PVC's only writable as root https://github.com/kubernetes/minikube/issues/1990 + - name: volume-permission-besu + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 /data"] + volumeMounts: + - name: data + mountPath: /data + securityContext: + runAsUser: 0 +{{- end}} + + containers: + +{{- if .Values.quorumFlags.privacy }} + - name: {{ .Release.Name }}-tessera + image: {{ .Values.image.tessera.repository }}:{{ .Values.image.tessera.tag }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + resources: + requests: + cpu: "{{ .Values.node.tessera.resources.cpuRequest }}" + memory: "{{ .Values.node.tessera.resources.memRequest }}" + limits: + cpu: "{{ .Values.node.tessera.resources.cpuLimit }}" + memory: "{{ .Values.node.tessera.resources.memLimit }}" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: TESSERA_CONFIG_TYPE + value: "-09" + volumeMounts: +{{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + - name: secrets-store + mountPath: {{ .Values.node.tessera.keysPath }} + readOnly: true +{{- else }} + - name: tessera-keys + mountPath: {{ .Values.node.tessera.keysPath }} + readOnly: true +{{- end }} + - name: data + mountPath: {{ .Values.node.tessera.dataPath }} + - name: tessera-peers + mountPath: /config/tessera-peers + ports: + - name: tessera + containerPort: {{ .Values.node.tessera.port }} + protocol: TCP + - name: tessera-tp + containerPort: {{ .Values.node.tessera.tpport }} + protocol: TCP + - name: tessera-q2t + containerPort: {{ .Values.node.tessera.q2tport }} + protocol: TCP + command: + - /bin/sh + - -c + args: + - | + exec + + cp {{ .Values.node.tessera.keysPath }}/tm.* {{ .Values.node.tessera.dataPath }}/ ; + + cat < {{ .Values.node.tessera.dataPath }}/tessera-config-09.json + { + "mode": "orion", + "useWhiteList": false, + "jdbc": { + "username": "sa", + "password": "", + "url": "jdbc:h2:{{ .Values.node.tessera.dataPath }}/db;MODE=Oracle;TRACE_LEVEL_SYSTEM_OUT=0", + "autoCreateTables": true + }, + "serverConfigs":[ + { + "app":"ThirdParty", + "enabled": true, + "serverAddress": "http://{{ include "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.node.tessera.tpport }}", + "communicationType" : "REST" + }, + { + "app":"Q2T", + "enabled": true, + "serverAddress": "http://{{ include "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.node.tessera.q2tport }}", + "sslConfig": { + "tls": "OFF" + }, + "communicationType" : "REST" + }, + { + "app":"P2P", + "enabled": true, + "serverAddress": "http://{{ include "besu-node.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.node.tessera.port }}", + "sslConfig": { + "tls": "OFF" + }, + "communicationType" : "REST" + } + ], + "peer": $$(cat /config/tessera-peers/tesseraPeers), + "keys": { + {{ if .Values.node.tessera.password }} + "passwordFile": "{{ .Values.node.tessera.passwordPath }}", + {{ end }} + "keyData": [ + { + "privateKeyPath": "/keys/tm.key", + "publicKeyPath": "/keys/tm.pub" + } + ] + }, + "alwaysSendTo": [] + } + EOF + + cat {{ .Values.node.tessera.dataPath }}/tessera-config-09.json + /tessera/bin/tessera -configfile {{ .Values.node.tessera.dataPath }}/tessera-config-09.json + +{{- end }} + + + - name: {{ .Release.Name }}-besu + image: {{ .Values.image.besu.repository }}:{{ .Values.image.besu.tag }} + imagePullPolicy: {{ .Values.image.besu.pullPolicy }} + resources: + requests: + cpu: "{{ .Values.node.besu.resources.cpuRequest }}" + memory: "{{ .Values.node.besu.resources.memRequest }}" + limits: + cpu: "{{ .Values.node.besu.resources.cpuLimit }}" + memory: "{{ .Values.node.besu.resources.memLimit }}" + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{- if .Values.node.besu.envBesuOpts }} + - name: BESU_OPTS + value: "{{ .Values.node.besu.envBesuOpts }}" +{{- end }} +{{- if .Values.quorumFlags.usesBootnodes }} + - name: BESU_BOOTNODES + valueFrom: + configMapKeyRef: + name: besu-bootnodes + key: bootnodes-string +{{- end }} + volumeMounts: +{{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + - name: secrets-store + mountPath: {{ .Values.node.besu.keysPath }} + readOnly: true +{{- else }} + - name: besu-keys + mountPath: {{ .Values.node.besu.keysPath }} + readOnly: true +{{- if .Values.quorumFlags.privacy }} + - name: tessera-keys + mountPath: {{ .Values.node.besu.privacy.pubkeysPath }} + readOnly: true +{{- end }} +{{- end }} + - name: genesis + mountPath: /etc/genesis + readOnly: true + - name: static-nodes + mountPath: /config/static + - name: besu-config + mountPath: /etc/besu + readOnly: true + - name: data + mountPath: {{ .Values.node.besu.dataPath }} + ports: + - name: json-rpc + containerPort: {{ .Values.node.besu.rpc.port }} + protocol: TCP + - name: ws + containerPort: {{ .Values.node.besu.ws.port }} + protocol: TCP + - name: graphql + containerPort: {{ .Values.node.besu.graphql.port }} + protocol: TCP + - name: rlpx + containerPort: {{ .Values.node.besu.p2p.port }} + protocol: TCP + - name: discovery + containerPort: {{ .Values.node.besu.p2p.port }} + protocol: UDP + - name: metrics + containerPort: {{ .Values.node.besu.metrics.port }} + protocol: TCP + command: + - /bin/sh + - -c + args: + - | + exec + /opt/besu/bin/besu \ + --config-file=/etc/besu/config.toml \ + --Xdns-enabled=true --Xdns-update-enabled=true --Xnat-kube-service-name={{ include "besu-node.fullname" . }} \ + --min-gas-price=0 + + livenessProbe: + httpGet: + path: /liveness + port: 8545 + initialDelaySeconds: 180 + periodSeconds: 60 + volumes: + - name: genesis + configMap: + name: besu-genesis + items: + - key: genesis.json + path: genesis.json + - name: static-nodes + configMap: + name: besu-peers + items: + - key: static-nodes.json + path: static-nodes.json + - name: besu-config + configMap: + name: {{ include "besu-node.fullname" . }}-besu-config + +{{- if and (ne .Values.cluster.provider "local") (.Values.cluster.cloudNativeServices) }} + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: {{ include "besu-node.fullname" . }}-secret-provider +{{- else }} + - name: besu-keys + secret: + secretName: {{ include "besu-node.fullname" . }}-keys +{{- if .Values.quorumFlags.privacy }} + - name: tessera-keys + secret: + secretName: {{ include "besu-node.fullname" . }}-tessera-keys +{{- end }} +{{- end }} +{{- if .Values.quorumFlags.privacy }} + - name: tessera-peers + configMap: + name: tessera-peers + items: + - key: tesseraPeers + path: tesseraPeers +{{- end }} +{{- if .Values.node.besu.permissions.enabled }} + - name: permissions-config + configMap: + name: {{ include "besu-node.fullname" . }}-permissions +{{- end }} diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-storage.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-storage.yaml new file mode 100644 index 0000000..5a22368 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/node-storage.yaml @@ -0,0 +1,70 @@ +{{- if eq .Values.cluster.provider "azure" }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ include "besu-node.fullname" . }}-storage + namespace: {{ .Release.Namespace }} +provisioner: file.csi.azure.com # replace with "kubernetes.io/azure-file" if aks version is less than 1.21 +reclaimPolicy: {{ .Values.cluster.reclaimPolicy }} +allowVolumeExpansion: true +mountOptions: + - dir_mode=0755 + - file_mode=0755 + - uid=1000 + - gid=1000 + - mfsymlinks +parameters: + skuName: Standard_LRS + +{{- else if eq .Values.cluster.provider "aws" }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ include "besu-node.fullname" . }}-storage + namespace: {{ .Release.Namespace }} +provisioner: {{ .Values.storage.aws.provisioner }} +reclaimPolicy: {{ .Values.cluster.reclaimPolicy }} +allowVolumeExpansion: true +parameters: + type: {{ .Values.storage.aws.parameters.type }} + fsType: {{ .Values.storage.aws.parameters.fsType }} + +# --- +# apiVersion: storage.k8s.io/v1 +# kind: StorageClass +# metadata: +# name: {{ include "besu-node.fullname" . }}-storage +# namespace: {{ .Release.Namespace }} +# provisioner: efs.csi.aws.com +# reclaimPolicy: Retain +# parameters: +# provisioningMode: efs-ap +# fileSystemId: #your_file_system_id +# directoryPerms: "700" +# gidRangeStart: "1000" # optional +# gidRangeEnd: "2000" # optional +# basePath: "/dynamic_provisioning" # optional + +{{- else }} + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ include "besu-node.fullname" . }}-storage + namespace: {{ .Release.Namespace }} + labels: + type: local +spec: + persistentVolumeReclaimPolicy: {{ .Values.cluster.reclaimPolicy }} + storageClassName: {{ include "besu-node.fullname" . }}-storage + capacity: + storage: "{{ .Values.storage.sizeLimit }}" + accessModes: + - ReadWriteOnce + hostPath: + path: "/tmp/{{ include "besu-node.fullname" . }}" + +{{- end }} diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/templates/permissions-configmap.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/permissions-configmap.yaml new file mode 100644 index 0000000..fc16a07 --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/templates/permissions-configmap.yaml @@ -0,0 +1,20 @@ +{{- if .Values.node.besu.permissions.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "besu-node.fullname" . }}-permissions + labels: + app.kubernetes.io/name: besu-permissions + app.kubernetes.io/component: besu + app.kubernetes.io/part-of: {{ include "besu-node.fullname" . }} + app.kubernetes.io/namespace: {{ .Release.Namespace }} + app.kubernetes.io/release: {{ .Release.Name }} + app.kubernetes.io/managed-by: helm +data: + permissions_config.toml: |- + # Permissioning TOML file + + accounts-allowlist={{ .Values.node.besu.permissions.accounts.allowlist }} + nodes-allowlist={{ .Values.node.besu.permissions.nodes.allowlist }} + +{{- end -}} \ No newline at end of file diff --git a/src/eth/instance/infra/kubernetes/charts/besu-node/values.yaml b/src/eth/instance/infra/kubernetes/charts/besu-node/values.yaml new file mode 100644 index 0000000..b4581fa --- /dev/null +++ b/src/eth/instance/infra/kubernetes/charts/besu-node/values.yaml @@ -0,0 +1,158 @@ +--- + +quorumFlags: + privacy: false + # this will erase keys - so be careful with this on validators + removeKeysOnDelete: false + isBootnode: true # set this to true if this node is a bootnode + usesBootnodes: true # set this to true if the network you are connecting to use a bootnode/s that are deployed in the cluster + +cluster: + provider: local # choose from: local | aws | azure + cloudNativeServices: false # set to true to use Cloud Native Services (SecretsManager and IAM for AWS; KeyVault & Managed Identities for Azure) + reclaimPolicy: Delete # set to either Retain or Delete + +aws: + # the aws cli commands uses the name 'quorum-sa' so only change this if you altered the name + serviceAccountName: quorum-sa + # the region you are deploying to + region: ap-southeast-2 + +azure: + serviceAccountName: quorum-sa + # the clientId of the user assigned managed identity created in the template + identityClientId: azure-clientId + # the clientId of the user assigned managed identity in the node pool's resource group + nodePoolIdentityClientId: azure-clientId + keyvaultName: azure-keyvault + # the tenant ID of the key vault + tenantId: azure-tenantId + # the subscription ID to use - this needs to be set explicitly when using multi tenancy + subscriptionId: azure-subscriptionId + +storage: + sizeLimit: "20Gi" + pvcSizeLimit: "20Gi" + # NOTE: when you set this to Retain, the volume WILL persist after the chart is delete and you need to manually delete it + reclaimPolicy: "Delete" # choose from: Delete | Retain + aws: + provisioner: kubernetes.io/aws-ebs + parameters: + type: gp3 + fsType: ext4 + +# fixes permissions of volumes becuase besu runs as user `besu` and volumes prefer `root` +volumePermissionsFix: + - local + - aws + +node: + besu: + envBesuOpts:"" + resources: + cpuLimit: 0.7 + cpuRequest: 0.5 + memLimit: "2G" + memRequest: "1G" + # privKey: + # pubKey: + dataPath: "/data/besu" + keysPath: "/keys" + privateKeyPath: "/keys/nodekey" + genesisFilePath: "/etc/genesis/genesis.json" + logging: INFO + customLabels: {} + account: + password: 'password' + passwordPath: "/keys/accountPassword" + p2p: + enabled: true + host: "0.0.0.0" + port: 30303 + discovery: true + staticNodes: "/config/static/static-nodes.json" + maxPeers: 25 + rpc: + enabled: true + host: "0.0.0.0" + port: 8545 + api: '["DEBUG","ETH", "ADMIN", "WEB3", "IBFT", "NET", "TRACE", "EEA", "PRIV", "QBFT", "PERM", "TXPOOL"]' + corsOrigins: '["all"]' + authenticationEnabled: false + # Number of allowed active RPC connections for Besu node + maxActiveConnections: 80 + ws: + enabled: true + host: "0.0.0.0" + port: 8546 + api: '["DEBUG","ETH", "ADMIN", "WEB3", "IBFT", "NET", "TRACE", "EEA", "PRIV", "QBFT", "PERM", "TXPOOL"]' + authenticationEnabled: false + graphql: + enabled: false + host: "0.0.0.0" + port: 8547 + corsOrigins: '["all"]' + http: + allowlist: '["*"]' + metrics: + enabled: true + host: "0.0.0.0" + port: 9545 + # enable if using prometheus-stack metrics monitoring + serviceMonitorEnabled: false + metricsPush: + enabled: false + host: "5.5.5.5" + port: 9091 + interval: 15 + prometheusJob: "besu" + privacy: + url: "http://localhost:9101" + pubkeysPath: "/tessera" + pubkeyFile: "tm.pub" + onchainGroupsEnabled: false + permissions: + enabled: false + filePath: "" + nodes: + enabled: false + allowlist: ["enode://abcd..@1.2.3.3:30303","enode://efba..@1.2.3.4:besu-node-bootnode-1-0.besu-node-bootnode-1.besu.svc.cluster.local:30303"] + accounts: + enabled: false + allowlist: ["0x0000000000000000000000000000000000008888"] + nodesContract: + enabled: false + address: "0x0000000000000000000000000000000000009999" + accountsContract: + enabled: false + address: "0x0000000000000000000000000000000000008888" + + tessera: + resources: + cpuLimit: 1 + cpuRequest: 0.5 + memLimit: "2G" + memRequest: "1G" + tmkey: "" + tmpub: "" + password: "password" + passwordPath: "/keys/tm.password" + dataPath: "/data/tessera" + keysPath: "/keys" + port: 9000 + tpport: 9080 + q2tport: 9101 + +image: + besu: + repository: hyperledger/besu + tag: 24.7. + pullPolicy: IfNotPresent + tessera: + repository: quorumengineering/tessera + tag: 22.1.7 + pullPolicy: IfNotPresent + hooks: + repository: consensys/quorum-k8s-hooks + tag: qgt-0.2.15 + pullPolicy: IfNotPresent \ No newline at end of file