backport: assumeutxo M5 — loadtxoutset end-to-end (bitcoin#27596 + cluster + E2E)

contrib/devtools/test_utxo_snapshots.sh

@@ -0,0 +1,203 @@
+#!/usr/bin/env bash
+# Demonstrate the creation and usage of UTXO snapshots.
+#
+# A server node starts up, IBDs up to a certain height, then generates a UTXO
+# snapshot at that point.
+#
+# The server then downloads more blocks (to create a diff from the snapshot).
+#
+# We bring a client up, load the UTXO snapshot, and we show the client sync to
+# the "network tip" and then start a background validation of the snapshot it
+# loaded. We see the background validation chainstate removed after validation
+# completes.
+#
+
+export LC_ALL=C
+set -e
+
+BASE_HEIGHT=${1:-30000}
+INCREMENTAL_HEIGHT=20000
+FINAL_HEIGHT=$(($BASE_HEIGHT + $INCREMENTAL_HEIGHT))
+
+SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT"
+CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT"
+UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat"
+
+# Chosen to try to not interfere with any running dashd processes.
+SERVER_PORT=8633
+SERVER_RPC_PORT=8632
+
+CLIENT_PORT=8733
+CLIENT_RPC_PORT=8732
+
+SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}"
+CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}"
+
+# Ensure the client exercises all indexes to test that snapshot use works
+# properly with indexes.
+ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1"
+
+if ! command -v jq >/dev/null ; then
+  echo "This script requires jq to parse JSON RPC output. Please install it."
+  echo "(e.g. sudo apt install jq)"
+  exit 1
+fi
+
+DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json"
+
+finish() {
+  echo
+  echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs"
+  echo
+  rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT"
+  rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR"
+  kill -9 "$SERVER_PID" "$CLIENT_PID"
+}
+
+trap finish EXIT
+
+# Need to specify these to trick client into accepting server as a peer
+# it can IBD from, otherwise the default values prevent IBD from the server node.
+EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00"
+
+server_rpc() {
+  ./src/dash-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@"
+}
+client_rpc() {
+  ./src/dash-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@"
+}
+server_sleep_til_boot() {
+  while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+client_sleep_til_boot() {
+  while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+
+mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR"
+
+echo "Hi, welcome to the assumeutxo demo/test"
+echo
+echo "We're going to"
+echo
+echo "  - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}"
+echo "  - create a UTXO snapshot at that height"
+echo "  - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that"
+echo
+echo "then we'll demonstrate assumeutxo by "
+echo
+echo "  - starting another node (the 'client') and loading the snapshot in"
+echo "    * first you'll have to modify the code slightly (chainparams) and recompile"
+echo "    * don't worry, we'll make it easy"
+echo "  - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server"
+echo "  - observing the client validate the snapshot chain via background IBD"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- Starting the demo. You might want to run the two following commands in"
+echo "   separate terminal windows:"
+echo
+echo "   watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log"
+echo "   watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..."
+./src/dashd -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null
+
+echo
+echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..."
+sleep 2
+./src/dashd -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null &
+SERVER_PID="$!"
+
+server_sleep_til_boot
+server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT"
+cat "$DUMP_OUTPUT"
+kill -9 "$SERVER_PID"
+
+RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT")
+RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT")
+RPC_EVO=$(jq -r .evo_hash < "$DUMP_OUTPUT")
+RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT")
+RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT")
+
+# Wait for server to shutdown...
+while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+
+echo
+echo "-- Now: add the following to CMainParams::m_assumeutxo_data"
+echo "   in src/chainparams.cpp, and recompile:"
+echo
+echo "   {.height = ${RPC_BASE_HEIGHT}, .hash_serialized = AssumeutxoHash{uint256S(\"0x${RPC_AU}\")},"
+echo "    .evo_hash = EvoSnapshotHash{uint256S(\"0x${RPC_EVO}\")}, .nChainTx = ${RPC_NCHAINTX},"
+echo "    .blockhash = uint256S(\"0x${RPC_BLOCKHASH}\")},"
+echo
+echo
+echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..."
+./src/dashd $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \
+    $EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null
+
+echo
+echo "-- Starting the server node to provide blocks to the client node..."
+./src/dashd $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \
+    $EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null &
+SERVER_PID="$!"
+server_sleep_til_boot
+
+echo
+echo "-- Okay, what you're about to see is the client starting up and activating the snapshot."
+echo "   I'm going to display the top 14 log lines from the client on top of an RPC called"
+echo "   getchainstates, which is like getblockchaininfo but for both the snapshot and "
+echo "   background validation chainstates."
+echo
+echo "   You're going to first see the snapshot chainstate sync to the server's tip, then"
+echo "   the background IBD chain kicks in to validate up to the base of the snapshot."
+echo
+echo "   Once validation of the snapshot is done, you should see log lines indicating"
+echo "   that we've deleted the background validation chainstate."
+echo
+echo "   Once everything completes, exit the watch command with CTRL+C."
+echo
+read -p "When you're ready for all this, hit [enter]" _
+
+echo
+echo "-- Starting the client node to get headers from the server, then load the snapshot..."
+./src/dashd $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \
+    -connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+echo
+echo "-- Initial state of the client:"
+client_rpc getchainstates
+
+echo
+echo "-- Loading UTXO snapshot into client..."
+client_rpc loadtxoutset "$UTXO_DAT_FILE"
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/dash-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads "
+echo "   as the main chain properly..."
+echo
+echo "   Press CTRL+C after you're satisfied to exit the demo"
+echo
+read -p "Press [enter] to continue"
+
+while kill -0 "$CLIENT_PID"; do
+    sleep 1
+done
+./src/dashd $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \
+    -addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/dash-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Done!"

contrib/devtools/utxo_snapshot.sh

@@ -97,7 +97,7 @@ ${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
 
 if [[ "${OUTPUT_PATH}" = "-" ]]; then
   (>&2 echo "Generating txoutset info...")
-  ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
+  ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_3 | sed 's/^.*: "\(.\+\)\+",/\1/g'
 else
   (>&2 echo "Generating UTXO snapshot...")
   ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"

doc/design/assumeutxo.md

@@ -1,10 +1,46 @@
 # assumeutxo
 
 Assumeutxo is a feature that allows fast bootstrapping of a validating dashd
-instance with a very similar security model to assumevalid.
+instance.
 
-The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to
-respectively generate and load UTXO snapshots. The utility script
+## Loading a snapshot
+
+There is currently no canonical source for snapshots, but any downloaded snapshot
+will be checked against a hash that's been hardcoded in source code.
+
+Once you've obtained the snapshot, you can use the RPC command `loadtxoutset` to
+load it.
+
+### Pruning
+
+A pruned node can load a snapshot. To save space, it's possible to delete the
+snapshot file as soon as `loadtxoutset` finishes.
+
+The minimum `-dbcache` setting is 550 MiB, but this functionality ignores that
+minimum and uses at least 1100 MiB.
+
+As the background sync continues there will be temporarily two chainstate
+directories, each multiple gigabytes in size (likely growing larger than the
+the downloaded snapshot).
+
+### Indexes
+
+Indexes work but don't take advantage of this feature. They always start building
+from the genesis block. Once the background validation reaches the snapshot block,
+indexes will continue to build all the way to the tip.
+
+For indexes that support pruning, note that no pruning will take place between
+the snapshot and the tip, until the background sync has completed - after which
+everything is pruned. Depending on how old the snapshot is, this may temporarily
+use a significant amount of disk space.
+
+## Generating a snapshot
+
+The RPC command `dumptxoutset` can be used to generate a snapshot. This can be used
+to create a snapshot on one node that you wish to load on another node.
+It can also be used to verify the hardcoded snapshot hash in the source code.
+
+The utility script
 `./contrib/devtools/utxo_snapshot.sh` may be of use.
 
 ## General background

doc/release-notes-27596.md

@@ -0,0 +1,28 @@
+Pruning
+-------
+
+When using assumeutxo with `-prune`, the prune budget may be exceeded if it is set
+lower than 1100MB (i.e. `MIN_DISK_SPACE_FOR_BLOCK_FILES * 2`). Prune budget is normally
+split evenly across each chainstate, unless the resulting prune budget per chainstate
+is beneath `MIN_DISK_SPACE_FOR_BLOCK_FILES` in which case that value will be used.
+
+RPC
+---
+
+`loadtxoutset` has been added, which allows loading a UTXO snapshot of the format
+generated by `dumptxoutset`. Once this snapshot is loaded, its contents will be
+deserialized into a second chainstate data structure, which is then used to sync to
+the network's tip.
+
+Meanwhile, the original chainstate will complete the initial block download process in
+the background, eventually validating up to the block that the snapshot is based upon.
+
+The result is a usable dashd instance that is current with the network tip in a
+matter of minutes rather than hours. UTXO snapshot are typically obtained via
+third-party sources (HTTP, torrent, etc.) which is reasonable since their contents
+are always checked by hash.
+
+You can find more information on this process in the `assumeutxo` design
+document (<https://github.com/dashpay/dash/blob/master/doc/design/assumeutxo.md>).
+
+`getchainstates` has been added to aid in monitoring the assumeutxo sync process.

doc/release-notes-28685.md

@@ -0,0 +1,4 @@
+RPC
+---
+
+The `hash_serialized_2` value has been removed from `gettxoutsetinfo` since the value it calculated contained a bug and did not take all data into account. It is superseded by `hash_serialized_3` which provides the same functionality but serves the correctly calculated hash.

doc/zmq.md

@@ -141,11 +141,11 @@ Where the 8-byte uints correspond to the mempool sequence number.
     | hashtx | <32-byte transaction hash in Little Endian> | <uint32 sequence number in Little Endian>
 
 
-`rawblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages).
+`rawblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages).
 
     | rawblock | <serialized block> | <uint32 sequence number in Little Endian>
 
-`hashblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages).
+`hashblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages).
 
     | hashblock | <32-byte block hash in Little Endian> | <uint32 sequence number in Little Endian>
 

src/Makefile.am

@@ -280,6 +280,7 @@ BITCOIN_CORE_H = \
   interfaces/node.h \
   interfaces/wallet.h \
   kernel/blockmanager_opts.h \
+  kernel/chain.h \
   kernel/chainstatemanager_opts.h \
   kernel/checks.h \
   kernel/coinstats.h \
@@ -565,6 +566,7 @@ libbitcoin_node_a_SOURCES = \
   instantsend/lock.cpp \
   instantsend/net_instantsend.cpp \
   instantsend/signing.cpp \
+  kernel/chain.cpp \
   kernel/checks.cpp \
   kernel/coinstats.cpp \
   kernel/context.cpp \
@@ -1288,6 +1290,7 @@ libdashkernel_la_SOURCES = \
   init/common.cpp \
   instantsend/db.cpp \
   instantsend/instantsend.cpp \
+  kernel/chain.cpp \
   kernel/checks.cpp \
   kernel/coinstats.cpp \
   kernel/context.cpp \

src/chain.h

@@ -260,8 +260,12 @@ class CBlockIndex
      *
      * Does not imply the transactions are consensus-valid (ConnectTip might fail)
      * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true)
+     *
+     * Note that this will be true for the snapshot base block, if one is loaded (and
+     * all subsequent assumed-valid blocks) since its nChainTx value will have been set
+     * manually based on the related AssumeutxoData entry.
      */
-    bool HaveTxsDownloaded() const { return nChainTx != 0; }
+    bool HaveNumChainTxs() const { return nChainTx != 0; }
 
     int64_t GetBlockTime() const
     {

src/chainlock/handler.cpp

@@ -210,8 +210,9 @@ void ChainlockHandler::AcceptedBlockHeader(const CBlockIndex* pindexNew)
     m_chainlocks.AcceptedBlockHeader(pindexNew);
 }
 
-void ChainlockHandler::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex)
+void ChainlockHandler::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex)
 {
+    if (role == ChainstateRole::BACKGROUND) return;
     if (!m_mn_sync.IsBlockchainSynced()) {
         return;
     }

src/chainlock/handler.h

@@ -100,7 +100,7 @@ class ChainlockHandler final : public CValidationInterface
     void TransactionAddedToMempool(const CTransactionRef& tx, int64_t nAcceptTime, uint64_t mempool_sequence) override
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
-    void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) override
+    void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) override
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
 private:

src/chainlock/signing.cpp

@@ -185,8 +185,9 @@ void ChainLockSigner::BlockDisconnected(const std::shared_ptr<const CBlock>& blo
 }
 
 
-void ChainLockSigner::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
+void ChainLockSigner::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
 {
+    if (role == ChainstateRole::BACKGROUND) return;
     if (!m_mn_sync.IsBlockchainSynced()) {
         return;
     }

src/chainlock/signing.h

@@ -71,7 +71,7 @@ class ChainLockSigner final : public llmq::CRecoveredSigsListener, public CValid
     void UnregisterRecoveryInterface();
 
     // implements validation interface:
-    void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override
+    void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override
         EXCLUSIVE_LOCKS_REQUIRED(!cs_signer);
     void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override
         EXCLUSIVE_LOCKS_REQUIRED(!cs_signer);