From 78f61b018c687136239c03b1d5a65ddb0b131ea3 Mon Sep 17 00:00:00 2001 From: Yorick Date: Tue, 18 Jun 2024 16:44:07 +0000 Subject: [PATCH] Nethermind path state --- ethd | 13 +++++++++-- nethermind/docker-entrypoint.sh | 41 +++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/ethd b/ethd index c53e36be..da7e3e34 100755 --- a/ethd +++ b/ethd @@ -454,7 +454,8 @@ check_disk_space() { echo "It should currently be auto-pruning, check logs with \"$__me logs -f --tail 500 execution | grep \ Full\". Free space:" else - echo "If the below reads above ${__safe_prune} GiB free, prune it with \"$__me prune-nethermind\"" + echo "You might be able to resync Nethermind for path storage if you haven't done so already, with \"$__me resync-execution\"." + echo "Also verify the amount of space your consensus layer client takes, and resync it as needed." fi echo __display_docker_dir @@ -462,7 +463,8 @@ Full\". Free space:" elif [[ "${value}" =~ "geth.yml" ]] && [[ "${__free_space}" -lt 104857600 ]]; then echo echo "You are running Geth and have less than 100 GiB of free disk space." - echo "You may resync from scratch to use PBSS and slow on-disk DB growth, with \"$__me resync-execution\"." + echo "You may resync from scratch to use PBSS and slow on-disk DB growth, if you haven't done so already, with \"$__me resync-execution\"." + echo "Also verify the amount of space your consensus layer client takes, and resync it as needed." echo __display_docker_dir __display_docker_volumes @@ -1589,6 +1591,12 @@ prune-nethermind() { var="NETWORK" NETWORK=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + if [ -n "$(dodocker run --rm -v "$(dodocker volume ls -q -f "name=nm-eth1-data")":"/var/lib/nethermind" \ + alpine:3 ls "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState")" ]; then + echo "This Nethermind uses path-based storage. Manual pruning is not required." + return 0 + fi + if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then __min_free=262144000 __min_gib=250 @@ -1649,6 +1657,7 @@ prune-nethermind() { echo "Nethermind should auto-prune below ${threshold} GiB free. Check logs with \"$__me logs -f --tail 500 \ execution | grep Full\" to see whether it is." fi + echo "Instead of pruning, consider resyncing Nethermind to use path-based storage, with \"$__me resync-execution\"." read -rp "WARNING - this will prune Nethermind's database in the background. Do you wish to continue? (No/Yes) " yn case $yn in [Yy][Ee][Ss] ) break;; diff --git a/nethermind/docker-entrypoint.sh b/nethermind/docker-entrypoint.sh index bef68cca..e5351d81 100755 --- a/nethermind/docker-entrypoint.sh +++ b/nethermind/docker-entrypoint.sh @@ -61,26 +61,33 @@ if [ "${ARCHIVE_NODE}" = "true" ]; then echo "Nethermind archive node without pruning" __prune="--Sync.DownloadBodiesInFastSync=false --Sync.DownloadReceiptsInFastSync=false --Sync.FastSync=false --Sync.SnapSync=false --Sync.FastBlocks=false --Pruning.Mode=None --Sync.PivotNumber=0" else - __parallel=$(($(nproc)/4)) - if [ "${__parallel}" -lt 2 ]; then - __parallel=2 - fi - __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}" - if [ "${AUTOPRUNE_NM}" = true ]; then - __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace" - if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then - __prune="${__prune} --Pruning.FullPruningThresholdMb=375810" - else - __prune="${__prune} --Pruning.FullPruningThresholdMb=51200" + # Fresh DB or already path based storage + if [[ ! -d "/var/lib/nethermind/nethermind_db/${NETWORK}" || -d "/var/lib/nethermind/nethermind_db/${NETWORK}/pathState" ]]; then + __prune="" + else # Hash, set pruning parameters + __parallel=$(($(nproc)/4)) + if [ "${__parallel}" -lt 2 ]; then + __parallel=2 fi - fi - if [ "${__memtotal}" -ge 30 ]; then + __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${__parallel}" + if [ "${AUTOPRUNE_NM}" = true ]; then + __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace" + if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then + __prune="${__prune} --Pruning.FullPruningThresholdMb=375810" + else + __prune="${__prune} --Pruning.FullPruningThresholdMb=51200" + fi + fi + if [ "${__memtotal}" -ge 30 ]; then # Investigating instability - #__prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" - __prune="${__prune} --Pruning.FullPruningMemoryBudgetMb=16384 --Init.StateDbKeyScheme=HalfPath" + #__prune="${__prune} --Pruning.CacheMb=4096 --Pruning.FullPruningMemoryBudgetMb=16384" + __prune="${__prune} --Pruning.FullPruningMemoryBudgetMb=16384" + fi + fi + if [ -n "${__prune}" ]; then + echo "Using pruning parameters:" + echo "${__prune}" fi - echo "Using pruning parameters:" - echo "${__prune}" fi # Word splitting is desired for the command line parameters