Merge remote-tracking branch 'origin/master' into HEAD

Author: tomt1664

contrib/merge-prs.sh

@@ -3,38 +3,58 @@
 export LC_ALL=C
 set -eo pipefail
 
+# Setup base branch and Bitcoin/Elements remote names.
 BASE_ORIG=merged-master
 BASE="${BASE_ORIG}"
 BITCOIN_UPSTREAM_REMOTE=bitcoin
-BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
-# ELEMENTS_UPSTREAM_REMOTE=upstream
-# ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"
+export BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
+ELEMENTS_UPSTREAM_REMOTE=upstream
+export ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"
 
+# START USER CONFIG:
+# Set your target upstream here
+TARGET_UPSTREAM=$BITCOIN_UPSTREAM
+TARGET_NAME="Bitcoin"
+PR_PREFIX="bitcoin/bitcoin"
+# TARGET_UPSTREAM=$ELEMENTS_UPSTREAM
+# TARGET_NAME="Elements"
+# PR_PREFIX="ElementsProject/elements"
+
+# Set your git worktree location here. This is where the merges will be done, and where you should checkout the merged-master branch.
+WORKTREE="/home/byron/code/elements-worktree"
+
+# Set your parallellism during build/test. You probably want as many cores as possible.
+# Parallel functional tests can somewhat exceed your core count, depends on the build machine CPU/RAM.
+PARALLEL_BUILD=23  # passed to make -j
+PARALLEL_TEST=46  # passed to test_runner.py --jobs
+PARALLEL_FUZZ=12  # passed to test_runner.py -j when fuzzing
+
+# Setup a ccache dir if necessary.
+#export CCACHE_DIR="/tmp/ccache"
+#export CCACHE_MAXSIZE="20G"
+
+# Set and export a WEBHOOK environment variable to a Discord webhook URL outside of this script to get notifications of progress and failures.
+
+# We don't currently fuzz during merging. Check fuzzing and CI after a merge run.
 # Replace this with the location where we should put the fuzz test corpus
-BITCOIN_QA_ASSETS="${HOME}/.tmp/bitcoin/qa-assets"
-FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"
-mkdir -p "$(dirname "${BITCOIN_QA_ASSETS}")"
-
-# BEWARE: On some systems /tmp/ gets periodically cleaned, which may cause
-#   random files from this directory to disappear based on timestamp, and
-#   make git very confused
-WORKTREE="${HOME}/.tmp/elements-merge-worktree"
-mkdir -p "${HOME}/.tmp"
-
-# These should be tuned to your machine; below values are for an 8-core
-#   16-thread macbook pro
-PARALLEL_BUILD=4  # passed to make -j
-PARALLEL_TEST=12  # passed to test_runner.py --jobs
-PARALLEL_FUZZ=8  # passed to test_runner.py -j when fuzzing
+# BITCOIN_QA_ASSETS="${HOME}/code/bitcoin/qa-assets"
+# FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"
 
+# END USER CONFIG
+
+# Script
 SKIP_MERGE=0
 DO_BUILD=1
 KEEP_GOING=1
+DO_TEST=1
+DO_FUZZ=0
+NUM=15
+COUNT=0
 
 if [[ "$1" == "setup" ]]; then
     echo "Setting up..."
     echo
-    git config remote.upstream.url >/dev/null || remote add upstream "https://github.com/ElementsProject/elements.git"
+    git config remote.upstream.url >/dev/null || git remote add upstream "https://github.com/ElementsProject/elements.git"
     git config remote.bitcoin.url >/dev/null || git remote add bitcoin "https://github.com/bitcoin/bitcoin.git"
     if git worktree list --porcelain | grep --silent prunable; then
         echo "You have stale git worktrees, please either fix them or run 'git worktree prune'."
@@ -45,14 +65,7 @@ if [[ "$1" == "setup" ]]; then
     echo "Fetching all remotes..."
     echo
     git fetch --all
-    echo
-    #echo "Cloning fuzz test corpus..."
-    #echo
-    #if [[ ! -d "${BITCOIN_QA_ASSETS}" ]]; then
-    #    cd "$(dirname ${BITCOIN_QA_ASSETS})" && git clone https://github.com/bitcoin-core/qa-assets.git
-    #fi
-    #echo
-    echo "Done! Remember to also check out merged-master, and push it back up when finished."
+    echo "Done! Remember to also checkout merged-master at ${WORKTREE}"
     exit 0
 elif [[ "$1" == "continue" ]]; then
     SKIP_MERGE=1
@@ -62,17 +75,35 @@ elif [[ "$1" == "list-only" ]]; then
     DO_BUILD=0
 elif [[ "$1" == "step" ]]; then
     KEEP_GOING=0
+elif [[ "$1" == "merge-only" ]]; then
+    SKIP_MERGE=0
+    KEEP_GOING=0
+    DO_BUILD=0
+    DO_TEST=0
 elif [[ "$1" == "step-continue" ]]; then
     SKIP_MERGE=1
     KEEP_GOING=0
+elif [[ "$1" == "step-test" ]]; then
+    SKIP_MERGE=1
+    KEEP_GOING=0
+    DO_BUILD=0
+elif [[ "$1" == "step-fuzz" ]]; then
+    SKIP_MERGE=1
+    KEEP_GOING=0
+    DO_BUILD=0
+    DO_TEST=0
+elif [[ "$1" == "analyze" ]]; then
+    DO_BUILD=0
+    DO_TEST=0
 else
-    echo "Usage: $0 <setup|list-only|go|continue|step|step-continue>"
+    echo "Usage: $0 <setup|list-only|go|continue|step|step-continue|analyze>"
     echo "    setup will configure your repository for the first run of this script"
     echo "    list-only will simply list all the PRs yet to be done"
     echo "    go will try to merge every PR, building/testing each"
     echo "    continue assumes the first git-merge has already happened, and starts with building"
     echo "    step will try to merge/build/test a single PR"
     echo "    step-continue assumes the first git-merge has already happened, and will try to build/test a single PR"
+    echo "    analyze will analyze the next $NUM PRs and count conflicts NB: DO NOT CTRL+C THIS PROCESS"
     echo
     echo "Prior to use, please create a git worktree for the elements repo at:"
     echo "    $WORKTREE"
@@ -101,54 +132,95 @@ if [[ "$SKIP_MERGE" == "1" ]]; then
 fi
 
 ## Get full list of merges
-# for elements
-# COMMITS=$(git -C "$WORKTREE" log "$ELEMENTS_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Elements %s')
-# for bitcoin
-COMMITS=$(git -C "$WORKTREE" log "$BITCOIN_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Bitcoin %s')
+COMMITS=$(git -C "$WORKTREE" log "$TARGET_UPSTREAM" --not $BASE --merges --first-parent --pretty="format:%ct %cI %h $TARGET_NAME %s")
 
-cd "$WORKTREE"
+cd "$WORKTREE" || exit 1
 
 VERBOSE=1
 
+echo start > merge.log
+
 quietly () {
     if [[ "$VERBOSE" == "1" ]]; then
-        "$@"
+	date | tee --append merge.log
+        time "$@" 2>&1 | tee --append merge.log
     else
         chronic "$@"
     fi
 }
 
+notify () {
+    local MESSAGE="$1"
+    local JSON="{\"content\": \"$MESSAGE\"}"
+    if [ -n "$WEBHOOK" ]; then
+        curl -d "$JSON" -H "Content-Type: application/json" "$WEBHOOK"
+    else
+        echo "$MESSAGE"
+    fi
+    if [[ "$2" == "1" ]]; then
+	    exit 1
+    fi
+}
+
 ## Sort by unix timestamp and iterate over them
-#echo "$ELT_COMMITS" "$BTC_COMMITS" | sort -n -k1 | while read line
 echo "$COMMITS" | tac | while read -r line
 do
-    echo
-    echo "=-=-=-=-=-=-=-=-=-=-="
-    echo
-
-    echo -e "$line"
     ## Extract data and output what we're doing
-    DATE=$(echo "$line" | cut -d ' ' -f 2)
     HASH=$(echo "$line" | cut -d ' ' -f 3)
     CHAIN=$(echo "$line" | cut -d ' ' -f 4)
-    PR_ID=$(echo "$line" | cut -d ' ' -f 6 | tr -d :)
-    PR_ID_ALT=$(echo "$line" | cut -d ' ' -f 8 | tr -d :)
+    PR_ID=$(echo "$line" | grep -o -P "#\d+")
 
-    if [[ "$PR_ID" == "pull" ]]; then
-    PR_ID="${PR_ID_ALT}"
-    fi
-    echo -e "$CHAIN PR \e[37m$PR_ID \e[33m$HASH\e[0m on \e[32m$DATE\e[0m "
+	GIT_HEAD=$(git rev-parse HEAD)
 
     ## Do it
     if [[ "$1" == "list-only" ]]; then
+        echo -e "$line"
         continue
     fi
+    if [[ "$1" == "analyze" ]]; then
+        COUNT=$((COUNT + 1))
+        # todo: count conflicts in critical files
+        # CRITICAL_FILES=("src/wallet/spend.h", "src/wallet/spend.cpp")
+        MERGE_FILE="/tmp/$HASH.merge"
+        DIFF_FILE="/tmp/$HASH.diff"
+        git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" > "$MERGE_FILE" || true
+        git -C "$WORKTREE" diff > "$DIFF_FILE"
+        git -C "$WORKTREE" reset --hard "$GIT_HEAD" > /dev/null
+        # FILES=$(grep "CONFLICT" "$MERGE_FILE")
+        NUM_FILES=$(grep -c "CONFLICT" "$MERGE_FILE")
+        NUM_CONFLICTS=$(grep -c "<<<<<<<" "$DIFF_FILE")
+        echo "$COUNT. Merge up to $PR_ID ($HASH) has $NUM_CONFLICTS conflicts in $NUM_FILES files."
+        if [[ "$COUNT" == "$NUM" ]]; then
+            exit 0
+        else
+            continue
+        fi
+    fi
+    notify "starting merge of $PR_ID"
+
+    # check for stoppers and halt if found
+    # a stopper is normally the PR after the version has been changed
+    # ie. the branch point we want to stop at for this version
+    STOPPERS=(
+        "#30716" # bitcoin v28
+        "#32041" # bitcoin v29
+    )
+    for STOPPER in "${STOPPERS[@]}"
+    do
+	if [[ "$PR_ID" == *"$STOPPER"* ]]; then
+		echo "Found $STOPPER in $PR_ID! Exiting."
+		notify "hit stopper, exiting"
+		exit 1
+	else
+		echo "Didn't find $STOPPER in $PR_ID. Continuing."
+	fi
+    done
 
     if [[ "$SKIP_MERGE" == "1" ]]; then
         echo -e "Continuing build of \e[37m$PR_ID\e[0m at $(date)"
     else
         echo -e "Start merge/build of \e[37m$PR_ID\e[0m at $(date)"
-        git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_ID)"
+        git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" || notify "fail merge" 1
     fi
 
     # FIXME: ELEMENTS, move make building process into CMake one?
@@ -164,35 +236,43 @@ do
         # The following is an expansion of `make check` that skips the libsecp
         # tests and also the benchmarks (though it does build them!)
         echo "Building"
-        quietly make -j"$PARALLEL_BUILD" -k
-#        quietly make -j1 check
-        echo "Linting"
-        quietly ./ci/lint/06_script.sh
+        quietly make -j"$PARALLEL_BUILD" -k || notify "fail build" 1
+        # todo: fix linting step
+        # echo "Linting"
+        # quietly ./ci/lint/06_script.sh || notify "fail lint"
+    fi
+
+    if [[ "$DO_TEST" == "1" ]]; then
         echo "Testing"
-        quietly ./src/qt/test/test_elements-qt
-        quietly ./src/test/test_bitcoin
-        quietly ./src/bench/bench_bitcoin
-        quietly ./test/util/bitcoin-util-test.py
-        quietly ./test/util/rpcauth-test.py
-        quietly make -C src/univalue/ check
+        quietly ./src/qt/test/test_elements-qt || notify "fail test qt" 1
+        quietly ./src/test/test_bitcoin || notify "fail test bitcoin" 1
+        quietly ./src/bench/bench_bitcoin || notify "fail test bench" 1
+        quietly ./test/util/test_runner.py || notify "fail test util" 1
+        quietly ./test/util/rpcauth-test.py || notify "fail test rpc" 1
         echo "Functional testing"
-        quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST"
+        quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST" || notify "fail test runner" 1
+    fi
+
+    if [[ "$DO_FUZZ" == "1" ]]; then
         echo "Cleaning for fuzz"
         quietly make distclean || true
         quietly git -C "$WORKTREE" clean -xf
         echo "Building for fuzz"
         quietly ./autogen.sh
         # TODO turn on `,integer` after this rebase
-        quietly ./configure --with-incompatible-bdb --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC=clang CXX=clang++
+        quietly ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC="ccache clang" CXX="ccache clang++"
         quietly make -j"$PARALLEL_BUILD" -k
         echo "Fuzzing"
-        quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}"
+        quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}" || notify "fail fuzz" 1
     fi
 
     if [[ "$KEEP_GOING" == "0" ]]; then
-        exit 1
+        notify "$PR_ID done, exiting"
+        exit 0
+    else
+        echo "$PR_ID done, continuing"
     fi
 
-#    bummer1.sh
     SKIP_MERGE=0
+    echo "end" >> merge.log
 done

doc/github-merge-script.md

@@ -24,8 +24,8 @@ github-merge.py <PR_NUMBER>
 For example for PR 1518 I see this output:
 
 ```
-ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
-* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
+ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
+* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
 
 Dropping you on a shell so you can try building/testing the merged source.
 Run 'git diff HEAD~' to show the changes being merged.
@@ -47,10 +47,10 @@ exit
 In our example this results in the following output:
 
 ```
-[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propogation
+[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propagation
  Date: Thu Jan 22 14:49:26 2026 +0200
-ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
-* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
+ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
+* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
 ACKs:
 * ACK 6c7788a; built and tested locally  (delta1)
 * ACK 6c7788adf373cd8f0dc5c4fe2b7674625631721e; successfully ran local tests (apoelstra)

src/init.cpp

@@ -1834,6 +1834,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
                   index_cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
     }
     LogInfo("* Using %.1f MiB for chain state database", kernel_cache_sizes.coins_db * (1.0 / 1024 / 1024));
+    LogInfo("ELIP 203 is%s active\n", (chainparams.GetAcceptUnlimitedIssuances())?"":" not");
 
     assert(!node.mempool);
     assert(!node.chainman);

src/kernel/chainparams.cpp

@@ -1293,6 +1293,7 @@ class CLiquidTestNetParams : public CCustomParams {
         default_signblockscript = "51210217e403ddb181872c32a0cd468c710040b2f53d8cac69f18dad07985ee37e9a7151ae";
         create_discount_ct = false;
         accept_discount_ct = true;
+        accept_unlimited_issuances = true;
         UpdateFromArgs(args);
         multi_data_permitted = true;
         SetGenesisBlock();
@@ -1402,7 +1403,7 @@ class CLiquidV1Params : public CChainParams {
 
         enforce_pak = true;
 
-        accept_unlimited_issuances = args.GetBoolArg("-acceptunlimitedissuances", false);
+        accept_unlimited_issuances = args.GetBoolArg("-acceptunlimitedissuances", true);
 
         multi_data_permitted = true;
         create_discount_ct = args.GetBoolArg("-creatediscountct", false);