Merge compact-blocks: BIP 152 relay, LevelDB pruning, verified cascade

BIP 152 compact blocks:
- SipHash-2-4 restored for short TX ID generation
- Full send/receive path with mempool-based block reconstruction
- getblocktxn/blocktxn fallback for missing transactions
- sendcmpct(false, 2) negotiation after verack
- wtxidrelay (BIP 339) peer state tracking
- 15 unit tests for compact block operations

LevelDB pruning (fixes 4GB RSS crash):
- Bounded load at startup: only newest chain_length*2+10 shares
- Evicted shares removed from LevelDB during run_think() trim
- SharechainStorage::remove_share() exposed to node layer

Safe verified↔chain cascade (fixes potential UAF):
- Deferred destruction: trim chain → cascade to verified → destroy
- ShareChain::trim() deferred_destroy parameter
- ShareChain::remove() owns_data parameter
- Matches Python p2pool node.py:355-398 pattern

308 tests passing. RSS stable at ~69MB (was crashing at 4068MB).

Author: frstrtr

.github/workflows/build.yml

@@ -84,7 +84,7 @@ jobs:
                     core_test sharechain_test share_test \
                     test_threading test_weights \
                     test_header_chain test_mempool test_template_builder \
-                    test_doge_chain \
+                    test_doge_chain test_compact_blocks \
                     test_phase0_live test_phase1_live test_phase2_live \
                     test_phase3_live test_phase4_embedded test_phase4_live \
             -j$(nproc)

docs/FUTURE.md

@@ -148,6 +148,43 @@ At 308K shares × ~4.5 KB/share = **~1.39 GB RAM**. Three-tier architecture:
 Epoch size: 720 shares (~3 hours). Each epoch summary replaces 720 full
 share objects (~216 KB) with one ~100-byte record — **2,160× compression**.
 
+#### P1: Compact Share Relay (BIP 152 Inspired)
+
+Applying compact block concepts to sharechain relay. Since all peers maintain
+similar PPLNS window state, redundancy across consecutive shares is extreme:
+
+**Problem:** Each share ~4.5 KB includes a full coinbase transaction with all
+PPLNS payout outputs. Consecutive shares have ~95% identical coinbase data
+(same output scripts, slightly different amounts).
+
+**Approach** — three complementary techniques:
+
+| Technique | Savings | How |
+|-----------|---------|-----|
+| **Short-ID transaction refs** | ~80% of tx data | SipHash-2-4 short IDs for share coinbase inputs/outputs, reconstruct from PPLNS state |
+| **Delta-encoded coinbases** | ~90% of coinbase | Only transmit changed amount deltas vs. previous share's coinbase |
+| **Prefilled-only new outputs** | Variable | New miners entering PPLNS window get full output; existing miners referenced by index |
+
+**Wire format sketch:**
+
+```
+CompactShare {
+    share_header      // ~200 bytes (same as today)
+    coinbase_delta    // varint-encoded amount deltas per output index
+    new_outputs[]     // PrefilledOutput { index, script, amount } for new PPLNS entrants
+    removed_indexes[] // output indexes dropped from PPLNS window
+    nonce             // SipHash key derivation (reuse BIP 152 pattern)
+}
+```
+
+**Expected compression:** ~4.5 KB → ~300-500 bytes per share relay (~90%).
+At 308K window and 30-second share period, reduces sustained relay bandwidth
+from ~150 bytes/s to ~15 bytes/s per peer.
+
+**Dependency:** Requires all peers to maintain synchronized PPLNS state to
+reconstruct full shares. Fallback: request full share via `getsharedata`
+(analogous to `getblocktxn`).
+
 #### P1: Adaptive Window Consensus Integration
 
 - Share validation must agree on `adaptive_chain_length` given the same
@@ -204,9 +241,12 @@ delivery to a configurable URL (`--block-webhook URL`).
 ### Historical Worker Stats DB — Not Started
 Persist per-worker statistics to SQLite with configurable retention.
 
-### Direct Peer Block Broadcast — Not Started
-Parallel P2P broadcast of found blocks to multiple full nodes to reduce
-orphan rate. High impact for solo miners.
+### Direct Peer Block Broadcast — In Progress (BIP 152)
+Compact block relay via BIP 152 (sendcmpct/cmpctblock/getblocktxn/blocktxn).
+Reduces block relay bandwidth by ~90-95% using SipHash-2-4 short transaction
+IDs. Foundation committed: data structures, SipHash, peer negotiation.
+Remaining: wire send path (BuildCompactBlock → cmpctblock), receive path
+(reconstruct from mempool), multi-peer broadcast.
 
 ---
 

src/btclibs/crypto/siphash.cpp

@@ -0,0 +1,173 @@
+// Copyright (c) 2016-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "siphash.h"
+
+#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
+
+#define SIPROUND do { \
+    v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; \
+    v0 = ROTL(v0, 32); \
+    v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
+    v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
+    v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; \
+    v2 = ROTL(v2, 32); \
+} while (0)
+
+CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
+{
+    v[0] = 0x736f6d6570736575ULL ^ k0;
+    v[1] = 0x646f72616e646f6dULL ^ k1;
+    v[2] = 0x6c7967656e657261ULL ^ k0;
+    v[3] = 0x7465646279746573ULL ^ k1;
+    count = 0;
+    tmp = 0;
+}
+
+CSipHasher& CSipHasher::Write(uint64_t data)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    assert(count % 8 == 0);
+
+    v3 ^= data;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= data;
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+
+    count += 8;
+    return *this;
+}
+
+CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+    uint64_t t = tmp;
+    uint8_t c = count;
+
+    while (size--) {
+        t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
+        c++;
+        if ((c & 7) == 0) {
+            v3 ^= t;
+            SIPROUND;
+            SIPROUND;
+            v0 ^= t;
+            t = 0;
+        }
+    }
+
+    v[0] = v0;
+    v[1] = v1;
+    v[2] = v2;
+    v[3] = v3;
+    count = c;
+    tmp = t;
+
+    return *this;
+}
+
+uint64_t CSipHasher::Finalize() const
+{
+    uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
+
+    uint64_t t = tmp | (((uint64_t)count) << 56);
+
+    v3 ^= t;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= t;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}
+
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val)
+{
+    /* Specialized implementation for efficiency */
+    uint64_t d = val.GetUint64(0);
+
+    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
+    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
+    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
+    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(1);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(2);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(3);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    v3 ^= ((uint64_t)4) << 59;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= ((uint64_t)4) << 59;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}
+
+uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra)
+{
+    /* Specialized implementation for efficiency */
+    uint64_t d = val.GetUint64(0);
+
+    uint64_t v0 = 0x736f6d6570736575ULL ^ k0;
+    uint64_t v1 = 0x646f72616e646f6dULL ^ k1;
+    uint64_t v2 = 0x6c7967656e657261ULL ^ k0;
+    uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d;
+
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(1);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(2);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = val.GetUint64(3);
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    d = (((uint64_t)36) << 56) | extra;
+    v3 ^= d;
+    SIPROUND;
+    SIPROUND;
+    v0 ^= d;
+    v2 ^= 0xFF;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    SIPROUND;
+    return v0 ^ v1 ^ v2 ^ v3;
+}

src/btclibs/crypto/siphash.h

@@ -0,0 +1,47 @@
+// Copyright (c) 2016-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_CRYPTO_SIPHASH_H
+#define BITCOIN_CRYPTO_SIPHASH_H
+
+#include <stdint.h>
+
+#include <core/uint256.hpp>
+
+/** SipHash-2-4 */
+class CSipHasher
+{
+private:
+    uint64_t v[4];
+    uint64_t tmp;
+    uint8_t count; // Only the low 8 bits of the input size matter.
+
+public:
+    /** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
+    CSipHasher(uint64_t k0, uint64_t k1);
+    /** Hash a 64-bit integer worth of data
+     *  It is treated as if this was the little-endian interpretation of 8 bytes.
+     *  This function can only be used when a multiple of 8 bytes have been written so far.
+     */
+    CSipHasher& Write(uint64_t data);
+    /** Hash arbitrary bytes. */
+    CSipHasher& Write(const unsigned char* data, size_t size);
+    /** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
+    uint64_t Finalize() const;
+};
+
+/** Optimized SipHash-2-4 implementation for uint256.
+ *
+ *  It is identical to:
+ *    SipHasher(k0, k1)
+ *      .Write(val.GetUint64(0))
+ *      .Write(val.GetUint64(1))
+ *      .Write(val.GetUint64(2))
+ *      .Write(val.GetUint64(3))
+ *      .Finalize()
+ */
+uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
+uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra);
+
+#endif // BITCOIN_CRYPTO_SIPHASH_H

src/c2pool/storage/sharechain_storage.cpp

@@ -155,10 +155,19 @@ bool SharechainStorage::has_share(const uint256& hash)
     if (!m_leveldb_store) {
         return false;
     }
-    
+
     return m_leveldb_store->has_share(hash);
 }
 
+bool SharechainStorage::remove_share(const uint256& hash)
+{
+    if (!m_leveldb_store) {
+        return false;
+    }
+
+    return m_leveldb_store->remove_share(hash);
+}
+
 void SharechainStorage::log_storage_stats()
 {
     if (!m_leveldb_store) {

src/c2pool/storage/sharechain_storage.hpp

@@ -98,6 +98,13 @@ class SharechainStorage {
      * @return True if share exists
      */
     bool has_share(const uint256& hash);
+
+    /**
+     * @brief Remove a share from storage
+     * @param hash Share hash to remove
+     * @return True if successfully removed
+     */
+    bool remove_share(const uint256& hash);
 
     /**
      * @brief Log storage statistics

src/core/uint256.hpp

@@ -63,6 +63,14 @@ class base_uint
        return &pn[0];
    }
 
+   /// Get a 64-bit chunk at position pos (0-3 for uint256).
+   /// Used by SipHash (BIP 152 compact blocks).
+   uint64_t GetUint64(int pos) const
+   {
+       return static_cast<uint64_t>(pn[pos * 2])
+            | (static_cast<uint64_t>(pn[pos * 2 + 1]) << 32);
+   }
+
    uint32_t* end()
    {
        return &pn[WIDTH];