diff --git a/dysnix/geth/Chart.yaml b/dysnix/geth/Chart.yaml index 950baa38..63682ac2 100644 --- a/dysnix/geth/Chart.yaml +++ b/dysnix/geth/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: geth description: Go-ethereum blockchain node Helm Chart -version: 1.0.10 +version: 1.0.11 appVersion: v1.13.2 keywords: diff --git a/dysnix/geth/templates/configmap-scripts.yaml b/dysnix/geth/templates/configmap-scripts.yaml index c94717a4..afc9302e 100644 --- a/dysnix/geth/templates/configmap-scripts.yaml +++ b/dysnix/geth/templates/configmap-scripts.yaml @@ -7,3 +7,13 @@ metadata: data: check-readiness.sh: |- {{- include (print $.Template.BasePath "/scripts/_check-readiness.tpl") . | nindent 4 }} + {{- if or .Values.syncToS3.enabled .Values.initFromS3.eanbled }} + init-from-s3.sh: |- + {{- include (print $.Template.BasePath "/scripts/_init-from-s3.tpl") . | nindent 4 }} + sync-to-s3.sh: |- + {{- include (print $.Template.BasePath "/scripts/_sync-to-s3.tpl") . | nindent 4 }} + s3-env.sh: |- + {{- include (print $.Template.BasePath "/scripts/_s3-env.tpl") . | nindent 4 }} + s3-cron.sh: |- + {{- include (print $.Template.BasePath "/scripts/_s3-cron.tpl") . | nindent 4 }} + {{- end }} diff --git a/dysnix/geth/templates/rbac.yaml b/dysnix/geth/templates/rbac.yaml new file mode 100644 index 00000000..975d4827 --- /dev/null +++ b/dysnix/geth/templates/rbac.yaml @@ -0,0 +1,31 @@ +{{- if .Values.syncToS3.enabled }} +{{- $fullName := include "geth.fullname" . }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName }} + labels: {{ include "geth.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - {{ $fullName }}-s3-config + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName }} + labels: {{ include "geth.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ $fullName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/dysnix/geth/templates/s3-configmap.yaml b/dysnix/geth/templates/s3-configmap.yaml new file mode 100644 index 00000000..128300ea --- /dev/null +++ b/dysnix/geth/templates/s3-configmap.yaml @@ -0,0 +1,13 @@ +{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "geth.fullname" . }}-s3-config +data: + DATA_DIR: /root/.ethereum + SYNC_TO_S3: "False" + S3_BASE_URL: {{ tpl .Values.s3config.baseUrl . }} + S3_CHAINDATA_URL: {{ tpl .Values.s3config.chaindataUrl . }} + S3_ANCIENT_URL: {{ tpl .Values.s3config.ancientUrl . }} + FORCE_INIT: {{ ternary "True" "False" .Values.initFromS3.force | quote }} +{{- end }} diff --git a/dysnix/geth/templates/s3-cronjob-rbac.yaml b/dysnix/geth/templates/s3-cronjob-rbac.yaml new file mode 100644 index 00000000..4f5545a8 --- /dev/null +++ b/dysnix/geth/templates/s3-cronjob-rbac.yaml @@ -0,0 +1,45 @@ +{{- if .Values.syncToS3.cronjob.enabled -}} +{{- $fullName := print (include "geth.fullname" .) "-s3-cronjob" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $fullName }} + labels: {{ include "geth.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName }} + labels: {{ include "geth.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - delete + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - {{ include "geth.fullname" . }}-s3-config + verbs: + - get + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName }} + labels: {{ include "geth.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ $fullName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ $fullName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/dysnix/geth/templates/s3-cronjob.yaml b/dysnix/geth/templates/s3-cronjob.yaml new file mode 100644 index 00000000..fce74121 --- /dev/null +++ b/dysnix/geth/templates/s3-cronjob.yaml @@ -0,0 +1,71 @@ +{{- if .Values.syncToS3.cronjob.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "geth.fullname" . }}-sync-to-s3 + labels: + {{- include "geth.labels" . | nindent 4 }} +spec: + {{- with .Values.syncToS3.cronjob }} + schedule: "{{ .schedule }}" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + jobTemplate: + metadata: + name: {{ include "geth.fullname" $ }}-sync-to-s3 + spec: + activeDeadlineSeconds: 60 + backoffLimit: 0 + template: + metadata: + labels: + {{- include "geth.labels" $ | nindent 12 }} + spec: + restartPolicy: OnFailure + {{- with .imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + serviceAccountName: {{ include "geth.fullname" $ }}-s3-cronjob + {{- with .podSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: enable-sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + {{- with .securityContext }} + securityContext: + {{- toYaml . | nindent 14 }} + {{- end }} + command: + - /bin/sh + - /scripts/s3-cron.sh + - enable_sync + - 5s + volumeMounts: + - name: scripts + mountPath: /scripts + {{- with .resources }} + resources: + {{- toYaml . | nindent 14 }} + {{- end }} + volumes: + - name: scripts + configMap: + name: {{ template "geth.fullname" $ }}-scripts + {{- end }} +{{- end }} diff --git a/dysnix/geth/templates/s3-secret.yaml b/dysnix/geth/templates/s3-secret.yaml new file mode 100644 index 00000000..a94b046b --- /dev/null +++ b/dysnix/geth/templates/s3-secret.yaml @@ -0,0 +1,10 @@ +{{- if or .Values.initFromS3.enabled .Values.syncToS3.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "geth.fullname" . }}-s3-secret +data: + S3_ENDPOINT_URL: {{ .Values.s3config.endpointUrl | toString | b64enc }} + AWS_ACCESS_KEY_ID: {{ .Values.s3config.accessKeyId | toString | b64enc }} + AWS_SECRET_ACCESS_KEY: {{ .Values.s3config.secretAccessKey | toString | b64enc }} +{{- end }} diff --git a/dysnix/geth/templates/scripts/_init-from-s3.tpl b/dysnix/geth/templates/scripts/_init-from-s3.tpl new file mode 100644 index 00000000..3c4ffad1 --- /dev/null +++ b/dysnix/geth/templates/scripts/_init-from-s3.tpl @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +# shellcheck disable=SC2086,SC3037 + +set -e + +. /scripts/s3-env.sh + +process_inputs() { + # download even if already initialized + if [ "$FORCE_INIT" = "True" ]; then + echo "Force init enabled, existing data will be deleted." + rm -f "$INITIALIZED_FILE" + fi + # check if we are already initialized + if [ -f "$INITIALIZED_FILE" ]; then + echo "Blockchain already initialized. Exiting..."; exit 0 + fi + # check for S3 credentials + if [ -z "$S3_ENDPOINT_URL" ] || [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "S3 credentials are not provided, exiting"; exit 1 + fi +} + +progress() { + remote_stats=$("$S5CMD" cat "s3://${STATS_URL}") + case $1 in + "start") + while true; do + inodes=$(df -Phi "$DATA_DIR" | tail -n 1 | awk '{print $3}') + size=$(df -P -BG "$DATA_DIR" | tail -n 1 | awk '{print $3}')G + echo -e "$(date -Iseconds) | SOURCE TOTAL ${remote_stats} | DST USED Inodes:\t${inodes} Size:\t${size}" + sleep 2 + done & + progress_pid=$! ;; + "stop") + kill "$progress_pid" + progress_pid=0 ;; + "*") + echo "Unknown arg" ;; + esac +} + +check_lockfile() { + if "$S5CMD" cat "s3://${LOCKFILE_URL}" >/dev/null 2>&1; then + echo "Found existing lockfile, snapshot might be corrupted. Aborting download.." + exit 1 + fi +} + +# stop all background tasks +interrupt() { + echo "Got interrupt signal, stopping..." + for i in "$@"; do kill $i; done +} + +sync() { + # cleanup data always, s5cmd does not support "true" sync, it does not save object's timestamps + # https://github.com/peak/s5cmd/issues/532 + echo "Cleaning up local data..." + rm -rf "$ANCIENT_DIR" + rm -rf "$CHAINDATA_DIR" + # recreate data directories + mkdir -p "$CHAINDATA_DIR" + mkdir -p "$ANCIENT_DIR" + + echo "Starting download data from S3..." + progress start + + # perform remote snapshot download and remove local objects which don't exist in snapshot + # run two jobs in parallel, one for chaindata, second for ancient data + time "$S5CMD" --stat sync $EXCLUDE_ANCIENT "s3://${CHAINDATA_URL}/*" "${CHAINDATA_DIR}/" >/dev/null & + download_chaindata=$! + time nice "$S5CMD" --stat sync --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "s3://${ANCIENT_URL}/*" "${ANCIENT_DIR}/" >/dev/null & + download_ancient=$! + + # handle interruption / termination + trap 'interrupt ${download_chaindata} ${download_ancient} ${progress_pid}' INT TERM + # wait for all syncs to complete + wait $download_chaindata $download_ancient + + progress stop + + # all done, mark as initialized + touch "$INITIALIZED_FILE" +} + + +main() { + process_inputs + check_lockfile + sync +} + +main \ No newline at end of file diff --git a/dysnix/geth/templates/scripts/_s3-cron.tpl b/dysnix/geth/templates/scripts/_s3-cron.tpl new file mode 100644 index 00000000..7ad08782 --- /dev/null +++ b/dysnix/geth/templates/scripts/_s3-cron.tpl @@ -0,0 +1,81 @@ +#!/usr/bin/env sh +# shellcheck disable=SC1083 + +MODE="$1" +WAIT_TIMEOUT="$2" +CONFIGMAP_NAME={{ include "geth.fullname" . }}-s3-config +KUBECTL=$(which kubectl) +PATCH_DATA="" +POD_NAME={{ include "geth.fullname" . }}-0 + +check_ret(){ + ret="$1" + msg="$2" + # allow to override exit code, default value is ret + exit_code=${3:-${ret}} + if [ ! "$ret" -eq 0 ]; then + echo "$msg" + echo "return code ${ret}, exit code ${exit_code}" + exit "$exit_code" + fi +} + +check_pod_readiness() { + # wait for pod to become ready + echo "$(date -Iseconds) Waiting ${WAIT_TIMEOUT} for pod ${1} to become ready ..." + "$KUBECTL" wait --timeout="$WAIT_TIMEOUT" --for=condition=Ready pod "$1" + check_ret $? "$(date -Iseconds) Pod ${1} is not ready, nothing to do, exiting" 0 + + # ensuring pod is not terminating now + # https://github.com/kubernetes/kubernetes/issues/22839 + echo "$(date -Iseconds) Checking for pod ${1} to not terminate ..." + deletion_timestamp=$("$KUBECTL" get -o jsonpath='{.metadata.deletionTimestamp}' pod "$1") + check_ret $? "$(date -Iseconds) Cannot get pod ${1}, abort" + + [ -z "$deletion_timestamp" ] + check_ret $? "$(date -Iseconds) Pod ${1} is terminating now, try another time" 1 +} + +enable_sync() { + echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to enable sync" + PATCH_DATA='{"data":{"SYNC_TO_S3":"True"}}' +} + +disable_sync() { + echo "$(date -Iseconds) Patching configmap ${CONFIGMAP_NAME} to disable sync" + PATCH_DATA='{"data":{"SYNC_TO_S3":"False"}}' +} + +patch_configmap() { + "$KUBECTL" patch configmap "$CONFIGMAP_NAME" --type merge --patch "$PATCH_DATA" + check_ret $? "$(date -Iseconds) Fatal: cannot patch configmap ${CONFIGMAP_NAME}, abort" +} + +delete_pod() { + echo "$(date -Iseconds) Deleting pod ${1} to trigger action inside init container ..." + # delete the pod to trigger action inside init container + "$KUBECTL" delete pod "$1" --wait=false + check_ret $? "$(date -Iseconds) Fatal: cannot delete pod ${1}, abort" + echo "$(date -Iseconds) Pod ${1} deleted successfully, exiting. Check pod logs after restart." +} + +main() { + case "$MODE" in + "enable_sync") + check_pod_readiness "$POD_NAME" + enable_sync + patch_configmap + delete_pod "$POD_NAME" + ;; + # intended to be run inside initContainer after successful sync + "disable_sync") + disable_sync + patch_configmap + ;; + "*") + check_ret 1 "$(date -Iseconds) Mode value \"$MODE\" is incorrect, abort" + ;; + esac +} + +main \ No newline at end of file diff --git a/dysnix/geth/templates/scripts/_s3-env.tpl b/dysnix/geth/templates/scripts/_s3-env.tpl new file mode 100644 index 00000000..e39756c5 --- /dev/null +++ b/dysnix/geth/templates/scripts/_s3-env.tpl @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +export S5CMD=/s5cmd + +# chaindata options +export EXCLUDE_ANCIENT="--exclude *.cidx --exclude *.ridx --exclude *.cdat --exclude *.rdat" +export EXCLUDE_CHAINDATA="--exclude *.ldb --exclude *.sst" + +# local directory structure config +export DATA_DIR="${DATA_DIR:-/root/.ethereum}" +export CHAINDATA_DIR="${CHAINDATA_DIR:-${DATA_DIR}/geth/chaindata}" +export ANCIENT_DIR="${ANCIENT_DIR:-${CHAINDATA_DIR}/ancient}" +export INITIALIZED_FILE="${DATA_DIR}/.initialized" + +# s3 directory structure config +export S3_BASE_URL="${S3_BASE_URL?S3_BASE_URL not provided.}" +export S3_CHAINDATA_URL="${S3_CHAINDATA_URL?S3_CHAINDATA_URL not provided.}" +export S3_ANCIENT_URL="${S3_ANCIENT_URL?S3_ANCIENT_URL not provided.}" +export S_COMPLETED="/completed" +export S_STATS="/stats" +export S_LOCKFILE="/lockfile" +export CHAINDATA_URL="${S3_BASE_URL}${S3_CHAINDATA_URL}" +export ANCIENT_URL="${S3_BASE_URL}${S3_ANCIENT_URL}" +export COMPLETED_URL="${S3_BASE_URL}${S_COMPLETED}" +export LOCKFILE_URL="${S3_BASE_URL}${S_LOCKFILE}" +export STATS_URL="${S3_BASE_URL}${S_STATS}" + +# download/upload options +export FORCE_INIT="${FORCE_INIT:-False}" diff --git a/dysnix/geth/templates/scripts/_sync-to-s3.tpl b/dysnix/geth/templates/scripts/_sync-to-s3.tpl new file mode 100644 index 00000000..b07d6365 --- /dev/null +++ b/dysnix/geth/templates/scripts/_sync-to-s3.tpl @@ -0,0 +1,71 @@ +#!/usr/bin/env sh +# shellcheck disable=SC2086,SC3037 + +set -e + +. /scripts/s3-env.sh + +process_inputs() { + # enable sync via env variable + if [ "$SYNC_TO_S3" != "True" ]; then + echo "Sync is not enabled in config, exiting" + exit 0 + fi + # check for S3 credentials + if [ -z "$S3_ENDPOINT_URL" ] || [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo "S3 credentials are not provided, exiting" + exit 1 + fi +} + +check_recent_init() { + # if node has been initialized from snapshot <30 mins ago, skip upload + is_recent=$(find "$INITIALIZED_FILE" -type f -mmin +30 | wc -l | tr -d '[:blank:]') + + if [ -f "$INITIALIZED_FILE" ] && [ "$is_recent" -eq 0 ]; then + echo "Node has been initialized recently, skipping the upload. Exiting..."; exit 0 + fi +} + +# stop all background processes +interrupt() { + echo "Got interrupt signal, stopping..." + for i in "$@"; do kill $i; done +} + +sync() { + # add lockfile while uploading + # shellcheck disable=SC3028 + echo "${HOSTNAME} $(date +%s)" | "$S5CMD" pipe "s3://${LOCKFILE_URL}" + + # perform upload of local data and remove destination objects which don't exist locally + # run two jobs in parallel, one for chaindata, second for ancient data + time "$S5CMD" --stat sync --delete $EXCLUDE_ANCIENT "${CHAINDATA_DIR}/" "s3://${CHAINDATA_URL}/" & + upload_chaindata=$! + time nice "$S5CMD" --stat sync --delete --part-size 200 --concurrency 2 $EXCLUDE_CHAINDATA "${ANCIENT_DIR}/" "s3://${ANCIENT_URL}/" & + upload_ancient=$! + + # handle interruption / termination + trap 'interrupt ${upload_chaindata} ${upload_ancient}' INT TERM + # wait for parallel upload to complete + wait $upload_chaindata $upload_ancient + + # mark upload as completed + date +%s | "$S5CMD" pipe "s3://${COMPLETED_URL}" + "$S5CMD" rm "s3://${LOCKFILE_URL}" +} + +update_stats() { + inodes=$(df -Phi "${DATA_DIR}" | tail -n 1 | awk '{print $3}') + size=$(df -P -BG "${DATA_DIR}" | tail -n 1 | awk '{print $3}')G + echo -ne "Inodes:\t${inodes} Size:\t${size}" | "$S5CMD" pipe "s3://${STATS_URL}" +} + +main() { + process_inputs + check_recent_init + sync + update_stats +} + +main \ No newline at end of file diff --git a/dysnix/geth/templates/statefulset.yaml b/dysnix/geth/templates/statefulset.yaml index 301715e8..acfd5098 100644 --- a/dysnix/geth/templates/statefulset.yaml +++ b/dysnix/geth/templates/statefulset.yaml @@ -15,6 +15,10 @@ spec: metadata: annotations: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if or .Values.syncToS3.enabled .Values.initFromS3.enabled }} + checksum/s3-secret: {{ include (print $.Template.BasePath "/s3-secret.yaml") . | sha256sum }} + checksum/s3-configmap: {{ include (print $.Template.BasePath "/s3-configmap.yaml") . | sha256sum }} + {{- end }} labels: {{- include "geth.selectorLabels" . | nindent 8 }} {{- with .Values.podStatusLabels }} @@ -43,8 +47,63 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.initContainers }} initContainers: + {{- if .Values.initFromS3.enabled }} + {{- with .Values.s3config }} + - name: init-from-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + command: + - sh + - /scripts/init-from-s3.sh + envFrom: + - configMapRef: + name: {{ include "geth.fullname" $ }}-s3-config + - secretRef: + name: {{ include "geth.fullname" $ }}-s3-secret + volumeMounts: + - name: scripts + mountPath: /scripts + - name: data + mountPath: /root/.ethereum + {{- end }} + {{- end }} + {{- if .Values.syncToS3.enabled }} + {{- with .Values.s3config }} + - name: sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + command: + - /bin/sh + - /scripts/sync-to-s3.sh + envFrom: + - configMapRef: + name: {{ include "geth.fullname" $ }}-s3-config + - secretRef: + name: {{ include "geth.fullname" $ }}-s3-secret + volumeMounts: + - name: scripts + mountPath: /scripts + - name: data + mountPath: /root/.ethereum + {{- end }} + {{- with .Values.syncToS3.cronjob }} + {{- if .enabled }} + - name: disable-sync-to-s3 + image: "{{ .image.repository }}:{{ .image.tag }}" + imagePullPolicy: {{ .image.pullPolicy | quote }} + command: + - /bin/sh + - /scripts/s3-cron.sh + - disable_sync + - 5s + volumeMounts: + - name: scripts + mountPath: /scripts + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.extraInitContainers }} {{- tpl (toYaml . | nindent 6) $ }} {{- end }} containers: diff --git a/dysnix/geth/values.yaml b/dysnix/geth/values.yaml index 69f2cb44..e1711797 100644 --- a/dysnix/geth/values.yaml +++ b/dysnix/geth/values.yaml @@ -50,7 +50,7 @@ command: [] extraArgs: [] ## Extra init containers, can be templated -initContainers: [] +extraInitContainers: [] # - name: dumpconfig # image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" # imagePullPolicy: "{{ .Values.image.pullPolicy }}" @@ -265,3 +265,41 @@ config: pprof: enabled: false port: 6061 + +s3config: + image: + repository: peakcom/s5cmd + tag: v2.2.2 + pullPolicy: IfNotPresent + # Any S3-compatible object storage service should be supported, but has only been tested with GCS. + # I.e. Amazon S3, MinIO, DigitalOcean Spaces, CloudFlare R2. + # endpointUrl: https://s3.amazonaws.com + endpointUrl: https://storage.googleapis.com + # Assuming your S3 bucket name is `my-snapshot-bucket` and base directory name is Helm release name + baseUrl: my-snapshot-bucket/{{ .Release.Name }} + # These are relative to baseUrl + chaindataUrl: /chaindata + ancientUrl: /ancient + # How to create access key + # AWS S3 https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html + # GCS https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create + accessKeyId: REPLACEME + secretAccessKey: REPLACEME + +initFromS3: + # enable initContainer + enabled: false + # download snapshot from S3 on every pod start + force: false + +syncToS3: + # enable initContainer (won't enable actual sync) + enabled: false + # restart pod and trigger sync to S3 inside initContainer by schedule + cronjob: + enabled: false + image: + repository: dysnix/kubectl + tag: v1.27 + pullPolicy: IfNotPresent + schedule: "0 2 * * *"