Support Nethermind FlatDB (#2568)

Author: yorickdowne

default.env

@@ -258,6 +258,9 @@ EL_NODE_TYPE=pre-merge-expiry
 # empty or "false" disables it, and any other string is assumed to be
 # exact parameters for Reth's "download" function
 RETH_SNAPSHOT=
+# Nethermind FlatDB. Possible values empty (don't use FlatDB), flat, or flatintrie, see https://github.com/NethermindEth/nethermind/releases/tag/1.37.0
+# FlatDB cannot be used with an archive node.
+NETHERMIND_FLATDB=
 # EraE URL, see https://github.com/eth-clients/e2store-format-specs/ and https://ethpandaops.io/data/history/
 # If this URL is provided, an EL that supports EraE import will use it when fresh syncing
 ERA_URL=
@@ -501,4 +504,4 @@ DOCKER_ROOT=/var/lib/docker
 DOCKER_SOCK=/var/run/docker.sock
 
 # Used by ethd update - please do not adjust
-ENV_VERSION=56
+ENV_VERSION=57

ethd

@@ -5,7 +5,7 @@ set -Eeuo pipefail
 __project_name="Eth Docker"
 __app_name="Ethereum node"
 __sample_service="consensus"
-__min_env_version=56
+__min_env_version=57
 __target_pg=18
 __current_docker=29
 __docker_exe="docker"
@@ -2948,6 +2948,14 @@ prune-nethermind() {
     exit 1
   fi
 
+  # Check for FlatDB
+  var="NETHERMIND_FLATDB"
+  __get_value_from_env "${var}" "${__env_file}" "__value"
+  if [[ -n "${__value}" ]]; then
+    echo "Nethermind is configured for FlatDB, which need not and cannot be pruned. Aborting."
+    exit 1
+  fi
+
   __get_docker_free_space
 
   var="NETWORK"

nethermind.yml

@@ -29,6 +29,7 @@ services:
       - EL_EXTRAS=${EL_EXTRAS:-}
       - NODE_TYPE=${EL_NODE_TYPE:-pre-merge-expiry}
       - AUTOPRUNE_NM=${AUTOPRUNE_NM:-true}
+      - NM_FLATDB=${NETHERMIND_FLATDB:-}
       - NETWORK=${NETWORK}
       # For Grandine plugin
       - COMPOSE_FILE=${COMPOSE_FILE}
@@ -77,16 +78,13 @@ services:
       - 0.0.0.0
       - --JsonRpc.EnginePort
       - ${EE_PORT:-8551}
-      - --JsonRpc.AdditionalRpcUrls=http://127.0.0.1:1337|http|admin
       - --JsonRpc.JwtSecretFile=/var/lib/nethermind/ee-secret/jwtsecret
       - --Metrics.Enabled
       - "true"
       - --Metrics.ExposeHost
       - 0.0.0.0
       - --Metrics.ExposePort
       - "6060"
-      - --Pruning.FullPruningCompletionBehavior
-      - AlwaysShutdown
       - --log
       - ${LOG_LEVEL}
     labels:

nethermind/docker-entrypoint.sh

@@ -49,6 +49,9 @@ if [[ -O /var/lib/nethermind/ee-secret/jwtsecret ]]; then
   chmod 666 /var/lib/nethermind/ee-secret/jwtsecret
 fi
 
+# This will be set by custom network code or named network node type code
+__prune=""
+
 if [[ "${NETWORK}" =~ ^https?:// ]]; then
   echo "Custom testnet at ${NETWORK}"
   repo=$(awk -F'/tree/' '{print $1}' <<< "${NETWORK}")
@@ -76,15 +79,39 @@ else
   __network="--config ${NETWORK}"
 fi
 
-if [[ ! "${NETWORK}" =~ ^https?:// && ! "${NODE_TYPE}" = "archive" ]]; then  # Only configure prune parameters for named networks
+if [[ "${NODE_TYPE}" = "archive" ]]; then
+  __flat=""
+else
+  case "${NM_FLATDB}" in
+    "")
+      __flat=""
+      ;;
+    flat)
+      echo "Enabling Nethermind FlatDB with Layout Flat"
+      #__flat="--FlatDb.Enabled=true --FlatDb.ImportFromPruningTrieState=true"
+      __flat="--FlatDb.Enabled=true"
+      ;;
+    flatintrie)
+      echo "Enabling Nethermind FlatDB with Layout FlatInTrie"
+      #__flat="--FlatDb.Enabled=true --FlatDb.ImportFromPruningTrieState=true --FlatDb.Layout=FlatInTrie"
+      __flat="--FlatDb.Enabled=true --FlatDb.Layout=FlatInTrie"
+      ;;
+    *)
+      __flat=""
+      echo "Unknown value ${NM_FLATDB} for \"NETHERMIND_FLATDB\". Continuing without FlatDB."
+      ;;
+  esac
+fi
+
+if [[ ! "${NETWORK}" =~ ^https?:// && "${NODE_TYPE}" != "archive" && -z "${__flat}" ]]; then  # Only configure prune parameters for named networks, non-archive and HalfPath DB
   memtotal=$(awk '/MemTotal/ {printf "%d", int($2/1024/1024)}' /proc/meminfo)
   parallel=$(($(nproc)/4))
   if [[ "${parallel}" -lt 2 ]]; then
     parallel=2
   fi
-  __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${parallel}"
+  __prune="--Pruning.FullPruningMaxDegreeOfParallelism=${parallel} --Pruning.FullPruningCompletionBehavior=AlwaysShutdown --JsonRpc.AdditionalRpcUrls=http://127.0.0.1:1337|http|admin"
   if [[ "${AUTOPRUNE_NM}" = true ]]; then
-    __prune="${__prune} --Pruning.FullPruningTrigger=VolumeFreeSpace"
+    __prune+=" --Pruning.FullPruningTrigger=VolumeFreeSpace"
     if [[ "${NETWORK}" =~ (mainnet|gnosis) ]]; then
       __prune+=" --Pruning.FullPruningThresholdMb=375810"
     else
@@ -153,6 +180,10 @@ esac
 
 echo "Using pruning parameters:"
 echo "${__prune}"
+if [[ -n "${__flat}" ]]; then
+  echo "Using FlatDB parameters:"
+  echo "${__flat}"
+fi
 
 # New or old datadir
 if [[ -d /var/lib/nethermind-og/nethermind_db ]]; then
@@ -318,4 +349,4 @@ set -- "${__args[@]}"
 
 # Word splitting is desired for the command line parameters
 # shellcheck disable=SC2086
-exec "$@" ${__datadir} ${__network} ${__prune} ${__grandine} ${EL_EXTRAS}
+exec "$@" ${__datadir} ${__network} ${__prune} ${__flat} ${__grandine} ${EL_EXTRAS}