dgb: per-job template-retention seam — captured_template_txs_fn provider (#271)

.github/workflows/build.yml

@@ -82,7 +82,11 @@ jobs:
                     test_address_resolution test_compute_share_target \
                     test_utxo test_dgb_subsidy test_dgb_coinbase_value dgb_share_test dgb_block_assembly_test dgb_header_sample_build_test dgb_header_ingest_test dgb_mempool_ingest_test \
                     dgb_gentx_coinbase_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test dgb_gentx_share_path_test dgb_other_tx_resolver_test \
+<<<<<<< HEAD
                     dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_coinbase_value_parity_test dgb_submit_classify_test dgb_aux_parent_coinbase_parity_test \
+=======
+                    dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_template_capture_test dgb_coinbase_value_parity_test dgb_submit_classify_test \
+>>>>>>> 38094399e (dgb: per-job template-retention seam — captured_template_txs_fn provider (#271))
                     rpc_request_test softfork_check_test genesis_check_test algo_select_test digishield_walk_test header_chain_test \
                     dgb_coin_node_seam_test dgb_block_broadcast_test dgb_won_block_dispatch_test dgb_forced_won_share_dualpath_test dgb_scrypt_pow_test dgb_nonce_grinder_test dgb_regrind_block_test dgb_won_block_finalize_test v37_test \
             -j$(nproc)
@@ -214,7 +218,11 @@ jobs:
                     test_address_resolution test_compute_share_target \
                     test_utxo test_dgb_subsidy test_dgb_coinbase_value dgb_share_test dgb_block_assembly_test dgb_header_sample_build_test dgb_header_ingest_test dgb_mempool_ingest_test \
                     dgb_gentx_coinbase_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test dgb_gentx_share_path_test dgb_other_tx_resolver_test \
+<<<<<<< HEAD
                     dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_coinbase_value_parity_test dgb_submit_classify_test dgb_aux_parent_coinbase_parity_test \
+=======
+                    dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_template_capture_test dgb_coinbase_value_parity_test dgb_submit_classify_test \
+>>>>>>> 38094399e (dgb: per-job template-retention seam — captured_template_txs_fn provider (#271))
                     rpc_request_test softfork_check_test genesis_check_test algo_select_test digishield_walk_test header_chain_test \
                     dgb_coin_node_seam_test dgb_block_broadcast_test dgb_won_block_dispatch_test dgb_forced_won_share_dualpath_test dgb_scrypt_pow_test dgb_nonce_grinder_test dgb_regrind_block_test dgb_won_block_finalize_test test_coin_broadcaster test_multiaddress_pplns test_pplns_stress \
                     v37_test \

src/impl/dgb/coin/template_capture.hpp

@@ -0,0 +1,135 @@
+// ---------------------------------------------------------------------------
+// template_capture.hpp -- the per-job template-retention seam (#271): the
+// PRODUCTION captured_template_txs_fn provider that feeds the won-block
+// reconstructor's template_other_txs_fn through the #299 deserialize bridge
+// (template_other_txs.hpp, make_template_other_txs_fn).
+//
+// WHY a capture store, not mempool re-selection:
+//   A won share commits to the GBT template it was MINED AGAINST at job hand-
+//   out. The reconstructed broadcast block's non-coinbase set MUST be that
+//   template's transactions[] in template order, because the share's merkle
+//   root was computed over [gentx] ++ those txs. main_dgb's interim
+//   template_other_txs_fn RE-SELECTS the live embedded mempool, which is
+//   merkle-consistent ONLY while the mempool is static across the won window
+//   (true under regtest, FALSE in prod: any mempool mutation between hand-out
+//   and won-block diverges the set -> wrong merkle root -> daemon-rejected
+//   block -> lost reward). This store removes that race: capture the template's
+//   transactions[] at hand-out keyed by the resulting share_hash, replay it
+//   verbatim at won-block.
+//
+// SSOT split: this header owns ONLY the keyed retain/replay + bounded eviction.
+// The transactions[] -> MutableTransaction decode is template_other_txs.hpp
+// (#299); compose make_template_other_txs_fn(capture.provider()) to get the
+// template_other_txs_fn make_reconstruct_closure_from_template installs.
+//
+// Miss policy: an absent share_hash yields an EMPTY transactions[] (json
+// ::array()) -> the bridge decodes to an empty other_txs -> a VALID coinbase-
+// only won block (the reconstructor's documented empty contract), NOT a fail-
+// closed nullopt. A won block that loses its fee txs is still a valid, network-
+// accepted block carrying the full subsidy; dropping it on a capture miss would
+// forfeit the subsidy too. Logged so a miss is never silent.
+//
+// Bounded: jobs are handed out continuously; the store keeps the most recent
+// `capacity` templates (FIFO eviction) so memory is O(capacity), not O(jobs).
+// A won share is reconstructed within a few jobs of hand-out, so a modest
+// capacity covers every realistic won-window.
+//
+// Per-coin isolation: src/impl/dgb/ only. p2pool-merged-v36 surface: NONE.
+// Thread-safe: capture() runs on the job/mint path, provider() fires on the
+// COMPUTE thread inside the won-block closure -- guarded by one mutex.
+// ---------------------------------------------------------------------------
+#pragma once
+
+#include <cstddef>
+#include <deque>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include <nlohmann/json.hpp>
+
+#include <core/uint256.hpp>
+
+namespace dgb
+{
+namespace coin
+{
+
+// Per-job captured-template store keyed by share_hash. Retains each handed-out
+// template's transactions[] (the conformant GBT shape: array of {data,txid,
+// hash,fee}) so the won-block reconstructor can replay the exact non-coinbase
+// set the share committed to.
+class TemplateCapture
+{
+public:
+    static constexpr std::size_t DEFAULT_CAPACITY = 256;
+
+    explicit TemplateCapture(std::size_t capacity = DEFAULT_CAPACITY)
+        : m_capacity(capacity == 0 ? 1 : capacity) {}
+
+    // Retain `transactions` (a GBT transactions[] array) for `share_hash`.
+    // Overwrites an existing entry for the same hash WITHOUT enqueuing the key
+    // twice (so eviction stays in true insertion order). FIFO-evicts the oldest
+    // once a new key would exceed capacity.
+    void capture(const uint256& share_hash, nlohmann::json transactions)
+    {
+        const std::string key = share_hash.GetHex();
+        std::lock_guard<std::mutex> lk(m_mtx);
+        auto it = m_store.find(key);
+        if (it == m_store.end())
+        {
+            if (m_store.size() >= m_capacity && !m_order.empty())
+            {
+                m_store.erase(m_order.front());
+                m_order.pop_front();
+            }
+            m_order.push_back(key);
+        }
+        m_store[key] = std::move(transactions);
+    }
+
+    // Replay the captured transactions[] for `share_hash`. Returns an empty
+    // json::array() on a miss (-> coinbase-only valid block), logged.
+    nlohmann::json provide(const uint256& share_hash) const
+    {
+        const std::string key = share_hash.GetHex();
+        std::lock_guard<std::mutex> lk(m_mtx);
+        auto it = m_store.find(key);
+        if (it == m_store.end())
+        {
+            std::cout << "[DGB-POOL-BLOCK] template-capture MISS for won share "
+                      << key.substr(0, 16)
+                      << " -- reconstructing coinbase-only (no retained template)"
+                      << std::endl;
+            return nlohmann::json::array();
+        }
+        return it->second;
+    }
+
+    // The captured_template_txs_fn make_template_other_txs_fn (#299) consumes.
+    // The returned closure captures `this`; the TemplateCapture MUST outlive the
+    // reconstruct closure it is installed into (it does in main_dgb: a run-loop-
+    // scoped member outliving the tracker callback).
+    std::function<nlohmann::json(const uint256&)> provider() const
+    {
+        return [this](const uint256& h) { return provide(h); };
+    }
+
+    std::size_t size() const
+    {
+        std::lock_guard<std::mutex> lk(m_mtx);
+        return m_store.size();
+    }
+
+private:
+    const std::size_t m_capacity;
+    mutable std::mutex m_mtx;
+    std::unordered_map<std::string, nlohmann::json> m_store;
+    std::deque<std::string> m_order;
+};
+
+} // namespace coin
+} // namespace dgb

src/impl/dgb/test/CMakeLists.txt

@@ -451,5 +451,24 @@ if (BUILD_TESTING AND GTest_FOUND)
         c2pool_payout c2pool_merged_mining c2pool_hashrate c2pool_storage
         dgb_coin pool sharechain)
     gtest_add_tests(dgb_aux_parent_coinbase_parity_test "" AUTO)
+    # --- #271 per-job template-retention seam (coin/template_capture.hpp) ------
+    # dgb_template_capture_test: pins the PRODUCTION captured_template_txs_fn the
+    # won-block reconstructor's template_other_txs_fn is built from (via the #299
+    # make_template_other_txs_fn bridge) -- capture(share,txs[]) -> provide(share)
+    # replays the exact template, a MISS replays empty (coinbase-only valid, NOT
+    # fail-closed), and composed through the bridge a HIT decodes to the same
+    # MutableTransaction vector deserialize_template_other_txs yields directly.
+    # Replaces main_dgb's interim mempool re-selection lambda (static-mempool-
+    # only). Compiles the tx codec + bridge, so it links the same embedded_tx_
+    # select SCC set as dgb_template_other_txs_test. MUST also be in BOTH build.yml
+    # --target allowlists (#143 NOT_BUILT trap).
+    add_executable(dgb_template_capture_test template_capture_test.cpp)
+    target_link_libraries(dgb_template_capture_test PRIVATE
+        GTest::gtest_main GTest::gtest
+        core dgb dgb_stratum
+        c2pool_payout c2pool_merged_mining c2pool_hashrate c2pool_storage
+        dgb_coin pool sharechain
+        nlohmann_json::nlohmann_json)
+    gtest_add_tests(dgb_template_capture_test "" AUTO)
 
 endif()

src/impl/dgb/test/template_capture_test.cpp

@@ -0,0 +1,172 @@
+// ---------------------------------------------------------------------------
+// dgb_template_capture_test -- pins coin/template_capture.hpp, the per-job
+// template-retention seam (#271): the PRODUCTION captured_template_txs_fn the
+// won-block reconstructor's template_other_txs_fn is built from (via the #299
+// make_template_other_txs_fn bridge), replacing main_dgb's interim mempool
+// RE-SELECTION lambda (merkle-consistent only on a static mempool).
+//
+// What it pins:
+//   * capture(share_hash, transactions[]) -> provide(share_hash) replays the
+//     SAME GBT transactions[] byte-faithfully (json-equal), keyed by share.
+//   * a capture MISS replays an empty array -> coinbase-only valid block (NOT
+//     fail-closed), so a missing template never forfeits the won subsidy.
+//   * make_template_other_txs_fn(capture.provider()) decodes a HIT to the exact
+//     MutableTransaction vector deserialize_template_other_txs yields directly
+//     from the same template -- i.e. the capture path is transparent to the
+//     #299 bridge -- and a MISS to an empty other_txs vector.
+//   * overwrite-same-hash keeps one entry (latest wins); FIFO eviction bounds
+//     the store and drops the OLDEST template once over capacity.
+//
+// Links the full dgb_coin codec + reconstruct closure (it composes the #299
+// bridge over the production make_mempool_tx_source template). MUST be in BOTH
+// build.yml --target allowlists (#143 NOT_BUILT trap). Per-coin isolation:
+// src/impl/dgb/ only.
+// ---------------------------------------------------------------------------
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include <impl/dgb/coin/template_capture.hpp>
+#include <impl/dgb/coin/template_other_txs.hpp>
+#include <impl/dgb/coin/embedded_tx_select.hpp>
+#include <impl/dgb/coin/mempool.hpp>
+#include <impl/dgb/coin/transaction.hpp>
+
+#include <core/pack.hpp>
+#include <core/uint256.hpp>
+#include <btclibs/util/strencodings.h>
+
+using dgb::coin::Mempool;
+using dgb::coin::MutableTransaction;
+using dgb::coin::TxIn;
+using dgb::coin::TxOut;
+using dgb::coin::TX_WITH_WITNESS;
+using dgb::coin::compute_txid;
+using dgb::coin::make_mempool_tx_source;
+using dgb::coin::deserialize_template_other_txs;
+using dgb::coin::make_template_other_txs_fn;
+using dgb::coin::TemplateCapture;
+
+namespace {
+
+MutableTransaction tagged_tx(int64_t value, uint32_t index)
+{
+    MutableTransaction tx;
+    tx.version = 1;
+    tx.locktime = 0;
+    TxIn in;
+    in.prevout.hash.SetNull();
+    in.prevout.index = index;
+    in.sequence = 0xffffffff;
+    tx.vin.push_back(in);
+    TxOut out;
+    out.value = value;
+    tx.vout.push_back(out);
+    return tx;
+}
+
+std::string withwit_hex(const MutableTransaction& tx)
+{
+    return HexStr(pack(TX_WITH_WITNESS(tx)).get_span());
+}
+
+uint256 hash_for(const char* hex)
+{
+    uint256 h; h.SetHex(hex);
+    return h;
+}
+
+// A production-shaped GBT transactions[] from a populated mempool.
+nlohmann::json sample_template_txs()
+{
+    Mempool pool;
+    MutableTransaction a = tagged_tx(11, 0);
+    MutableTransaction b = tagged_tx(22, 1);
+    pool.add_tx(a);
+    pool.add_tx(b);
+    pool.set_tx_fee(compute_txid(a), 700);
+    pool.set_tx_fee(compute_txid(b), 300);
+    return make_mempool_tx_source(pool, /*max_weight=*/4'000'000)().transactions;
+}
+
+const char* H_A = "00000000000000000000000000000000000000000000000000000000000000a1";
+const char* H_B = "00000000000000000000000000000000000000000000000000000000000000b2";
+const char* H_C = "00000000000000000000000000000000000000000000000000000000000000c3";
+const char* H_MISS = "00000000000000000000000000000000000000000000000000000000deadbeef";
+
+} // namespace
+
+// --- Test 1: capture -> provide round-trips the template byte-faithfully ------
+TEST(DgbTemplateCapture, CaptureProvideRoundTrip)
+{
+    TemplateCapture cap;
+    const auto txs = sample_template_txs();
+    ASSERT_EQ(txs.size(), 2u);
+
+    cap.capture(hash_for(H_A), txs);
+    EXPECT_EQ(cap.size(), 1u);
+    // json-equal: the exact transactions[] handed in comes back out.
+    EXPECT_EQ(cap.provide(hash_for(H_A)), txs);
+}
+
+// --- Test 2: a MISS replays an empty array (coinbase-only, not fail-closed) ---
+TEST(DgbTemplateCapture, MissReplaysEmptyArray)
+{
+    TemplateCapture cap;
+    cap.capture(hash_for(H_A), sample_template_txs());
+
+    const auto miss = cap.provide(hash_for(H_MISS));
+    ASSERT_TRUE(miss.is_array());
+    EXPECT_TRUE(miss.empty());
+}
+
+// --- Test 3: composed through the #299 bridge, a HIT decodes to the exact tx --
+// vector deserialize_template_other_txs yields directly from the same template;
+// a MISS decodes to an empty other_txs vector. Proves the capture path is
+// transparent to the reconstructor's template_other_txs_fn.
+TEST(DgbTemplateCapture, ComposedWithBridgeHitAndMiss)
+{
+    TemplateCapture cap;
+    const auto txs = sample_template_txs();
+    cap.capture(hash_for(H_A), txs);
+
+    auto other_txs_fn = make_template_other_txs_fn(cap.provider());
+
+    // HIT: identical to decoding the template directly, in template order.
+    const auto via_capture = other_txs_fn(hash_for(H_A));
+    const auto direct = deserialize_template_other_txs(txs);
+    ASSERT_EQ(via_capture.size(), direct.size());
+    ASSERT_EQ(via_capture.size(), 2u);
+    for (size_t i = 0; i < via_capture.size(); ++i)
+        EXPECT_EQ(withwit_hex(via_capture[i]), withwit_hex(direct[i]));
+
+    // MISS: empty other_txs -> the reconstructor frames a coinbase-only block.
+    EXPECT_TRUE(other_txs_fn(hash_for(H_MISS)).empty());
+}
+
+// --- Test 4: overwrite the same share_hash keeps one entry (latest wins) ------
+TEST(DgbTemplateCapture, OverwriteSameHashLatestWins)
+{
+    TemplateCapture cap;
+    cap.capture(hash_for(H_A), nlohmann::json::array());      // empty first
+    cap.capture(hash_for(H_A), sample_template_txs());        // then 2-tx template
+    EXPECT_EQ(cap.size(), 1u);                                // not duplicated
+    EXPECT_EQ(cap.provide(hash_for(H_A)).size(), 2u);         // latest content
+}
+
+// --- Test 5: FIFO eviction bounds the store, drops the OLDEST template --------
+TEST(DgbTemplateCapture, BoundedFifoEvictsOldest)
+{
+    TemplateCapture cap(/*capacity=*/2);
+    cap.capture(hash_for(H_A), sample_template_txs());
+    cap.capture(hash_for(H_B), sample_template_txs());
+    cap.capture(hash_for(H_C), sample_template_txs());        // evicts H_A
+
+    EXPECT_EQ(cap.size(), 2u);
+    EXPECT_TRUE(cap.provide(hash_for(H_A)).empty());          // oldest evicted -> miss
+    EXPECT_EQ(cap.provide(hash_for(H_B)).size(), 2u);         // newest retained
+    EXPECT_EQ(cap.provide(hash_for(H_C)).size(), 2u);
+}