Merge pull request #276 from dysnix/bsc-initial-pebble

[bsc] initial pebble, init-from-gcs improvements
dysnix · Dec 20, 2023 · e05b6a5 · e05b6a5
2 parents 9d73e72 + f24a67a
commit e05b6a5
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 19 deletions.
diff --git a/dysnix/bsc/Chart.yaml b/dysnix/bsc/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 name: bsc
 description: Binance Smart Chain chart for Kubernetes
-version: 0.6.36
+version: 0.6.37
 appVersion: 1.2.15
 
 keywords:

diff --git a/dysnix/bsc/templates/configs/_config.txt b/dysnix/bsc/templates/configs/_config.txt
@@ -5,6 +5,9 @@ NoPruning = false
 NoPrefetch = false
 TrieTimeout = 150000000000
 DisablePeerTxBroadcast = true
+{{- if .Values.bsc.state.scheme }}
+StateScheme = "{{ .Values.bsc.state.scheme }}"
+{{- end }}
 
 [Eth.Miner]
 GasCeil = 140000000

diff --git a/dysnix/bsc/templates/scripts/_init_from_gcs.tpl b/dysnix/bsc/templates/scripts/_init_from_gcs.tpl
@@ -11,9 +11,10 @@ CHAINDATA_DIR="${DATA_DIR}/geth/chaindata"
 INITIALIZED_FILE="${DATA_DIR}/.initialized"
 #without gs:// or s3://, just a bucket name and path
 INDEX_URL="{{ .Values.bsc.initFromGCS.indexUrl }}"
+GCS_BASE_URL="{{ .Values.bsc.initFromGCS.baseUrlOverride }}"
 S5CMD=/s5cmd
 EXCLUDE_ANCIENT="--exclude *.cidx --exclude *.ridx --exclude *.cdat --exclude *.rdat"
-EXCLUDE_STATE="--exclude *.ldb"
+EXCLUDE_STATE="--exclude *.ldb --exclude *.sst"
 INDEX="index"
 S_UPDATING="/updating"
 S_TIMESTAMP="/timestamp"
@@ -33,22 +34,24 @@ fi
 # we need to create temp files
 cd /tmp
 
-# get index of source base dirs
-${S5CMD} cp "s3://${INDEX_URL}" "${INDEX}"
-
-# get the most fresh datadir
-# prune time is ignored here, we assume that all datadirs are pruned frequently enough
-GCS_BASE_URL=""
-MAX_TIMESTAMP=1
-for _GCS_BASE_URL in $(cat ${INDEX});do
-  _TIMESTAMP_URL="${_GCS_BASE_URL}${S_TIMESTAMP}"
-  _TIMESTAMP=$(${S5CMD} cat s3://${_TIMESTAMP_URL})
-  if [ "${_TIMESTAMP}" -gt "${MAX_TIMESTAMP}" ];then
-    GCS_BASE_URL="${_GCS_BASE_URL}"
-    MAX_TIMESTAMP=${_TIMESTAMP}
-  fi
-done
-
+if [ "${GCS_BASE_URL}" == "" ];then
+  # get index of source base dirs
+  ${S5CMD} cp "s3://${INDEX_URL}" "${INDEX}"
+
+  # get the most fresh datadir
+  # prune time is ignored here, we assume that all datadirs are pruned frequently enough
+  MAX_TIMESTAMP=1
+  for _GCS_BASE_URL in $(cat ${INDEX});do
+    _TIMESTAMP_URL="${_GCS_BASE_URL}${S_TIMESTAMP}"
+    _TIMESTAMP=$(${S5CMD} cat s3://${_TIMESTAMP_URL})
+    if [ "${_TIMESTAMP}" -gt "${MAX_TIMESTAMP}" ];then
+      GCS_BASE_URL="${_GCS_BASE_URL}"
+      MAX_TIMESTAMP=${_TIMESTAMP}
+    fi
+  done
+else
+  echo "Using overridden base URL: ${GCS_BASE_URL}"
+fi
 if [ "${GCS_BASE_URL}" == "" ];then
   echo "Fatal: cannot pick up correct base url, exiting"
   exit 1

diff --git a/dysnix/bsc/templates/scripts/_sync_to_gcs.tpl b/dysnix/bsc/templates/scripts/_sync_to_gcs.tpl
@@ -27,7 +27,7 @@ RMLIST="${DATA_DIR}/rmlist.txt"
 
 # s5cmd excludes just by file extension, not by file path
 EXCLUDE_ANCIENT="--exclude *.cidx --exclude *.ridx --exclude *.cdat --exclude *.rdat"
-EXCLUDE_STATE="--exclude *.ldb"
+EXCLUDE_STATE="--exclude *.ldb --exclude *.sst"
 
 S_UPDATING="/updating"
 S_TIMESTAMP="/timestamp"

diff --git a/dysnix/bsc/templates/statefulset.yaml b/dysnix/bsc/templates/statefulset.yaml
@@ -133,6 +133,12 @@ spec:
           - --pprof.addr=0.0.0.0
           - --pprof.port={{ .Values.bsc.metrics.service.port }}
           {{- end }}
+          {{- if .Values.bsc.state.scheme }}
+          - --state.scheme={{ .Values.bsc.state.scheme }}
+          {{- end }}
+          {{- if .Values.bsc.db.engine }}
+          - --db.engine={{ .Values.bsc.db.engine }}
+          {{- end }}
           {{- range .Values.bsc.extraArgs }}
           - {{ . }}
           {{- end }}

diff --git a/dysnix/bsc/values.yaml b/dysnix/bsc/values.yaml
@@ -112,6 +112,14 @@ bsc:
   cache:
     value: 8192
     preimages: true
+  state:
+    # possible options (1.3.x+) are "hash" and "path"
+    # use empty value for 1.2.x
+    scheme: ""
+  db:
+    # possible options (1.3.x+) are "leveldb" and "pebble"
+    # use empty value for 1.2.x
+    engine: ""
   txlookuplimit: 0                        # WARNING: won't work on BSC >=1.3.x, node will crash if it's provided. use ".Values.history.transactions" instead
   history:
     transactions: null                    # BSC >=1.3.x only
@@ -138,6 +146,7 @@ bsc:
     keyID: "AWS_ACCESS_KEY_ID"
     accessKey: "AWS_SECRET_ACCESS_KEY"
     indexUrl: "bucket/path/to/file"
+    baseUrlOverride: ""                   # "bucket/path/to/dir"
     fullResyncOnSrcUpdate: false
   syncToGCS:
     enabled: false