Merge pull request #729 from frstrtr/fix/nmc-underfill-guard

nmc(underfill): port near-empty-template guard to NMC template_builder (+ KAT)

Author: frstrtr

.github/workflows/build.yml

@@ -123,7 +123,7 @@ jobs:
                     test_mweb_builder \
                     test_address_resolution test_compute_share_target test_web_honesty_regression \
                     test_utxo test_dgb_subsidy test_dgb_coinbase_value dgb_share_test dgb_redistribute_test dgb_block_assembly_test dgb_witness_commitment_test dgb_header_sample_build_test dgb_header_ingest_test dgb_mempool_ingest_test \
-                    dgb_gentx_coinbase_test dgb_connection_coinbase_test dgb_pplns_payout_split_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test nmc_mempool_name_test nmc_block_broadcast_test nmc_host_dualpath_test nmc_fallback_path_conformance_test dgb_gentx_share_path_test dgb_conn_pplns_producer_test dgb_other_tx_resolver_test \
+                    dgb_gentx_coinbase_test dgb_connection_coinbase_test dgb_pplns_payout_split_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_underfill_guard_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test nmc_mempool_name_test nmc_block_broadcast_test nmc_host_dualpath_test nmc_fallback_path_conformance_test dgb_gentx_share_path_test dgb_conn_pplns_producer_test dgb_other_tx_resolver_test \
                     dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_coinbase_value_parity_test dgb_submit_classify_test dgb_aux_parent_coinbase_parity_test dgb_template_capture_test dgb_aux_doge_db_commitment_bind_test dgb_aux_doge_mm_commitment_test dgb_aux_doge_dc_proof_test dgb_aux_doge_bind_parsers_test dgb_compact_blocks_bip152_parity_test dgb_aux_dual_target_select_test dgb_aux_broadcast_path_election_test dgb_aux_doge_submit_test dgb_aux_doge_embed_livewire_test dgb_aux_doge_dc_layout_verifier_test \
                     rpc_request_test softfork_check_test genesis_check_test algo_select_test digishield_walk_test header_chain_test \
                     dgb_coin_node_seam_test dgb_block_broadcast_test dgb_won_block_dispatch_test dgb_forced_won_share_dualpath_test dgb_scrypt_pow_test dgb_nonce_grinder_test dgb_regrind_block_test dgb_won_block_finalize_test dgb_share_target_genesis_test dgb_share_target_retarget_test dgb_share_bits_oracle_pin_test dgb_pool_msg_wire_test dgb_get_shares_walk_test dgb_download_stops_test dgb_think_p1_walk_bounds_test dgb_think_p1_desired_emit_test dgb_think_p6_desired_cutoff_test dgb_think_p4_head_keys_test dgb_think_p3_best_head_test dgb_g1_oracle_byte_parity_test dgb_think_p2_walk_bounds_test dgb_expected_time_to_block_test dgb_tail_score_endpoints_test dgb_pool_attempts_per_second_test dgb_pool_efficiency_test dgb_think_p5_best_share_punish_test dgb_auto_ratchet_tail_guard_test dgb_auto_ratchet_sim_test dgb_binomial_conf_interval_test dgb_desired_version_tally_test dgb_min_protocol_ratchet_test dgb_get_height_and_last_endpoints_test dgb_chain_walk_window_test dgb_redistribute_delegate_ghal_test dgb_share_weight_decay_test dgb_naughty_propagation_test dgb_hash_format_parity_test dgb_emergency_decay_saturation_test dgb_arith256_muldiv_kat_test v37_test \
@@ -280,7 +280,7 @@ jobs:
                     test_mweb_builder \
                     test_address_resolution test_compute_share_target test_web_honesty_regression \
                     test_utxo test_dgb_subsidy test_dgb_coinbase_value dgb_share_test dgb_redistribute_test dgb_block_assembly_test dgb_witness_commitment_test dgb_header_sample_build_test dgb_header_ingest_test dgb_mempool_ingest_test \
-                    dgb_gentx_coinbase_test dgb_connection_coinbase_test dgb_pplns_payout_split_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test nmc_mempool_name_test nmc_block_broadcast_test nmc_host_dualpath_test nmc_fallback_path_conformance_test dgb_gentx_share_path_test dgb_conn_pplns_producer_test dgb_other_tx_resolver_test \
+                    dgb_gentx_coinbase_test dgb_connection_coinbase_test dgb_pplns_payout_split_test nmc_auxpow_merkle_test nmc_template_builder_test nmc_underfill_guard_test nmc_auxpow_wire_test nmc_reconstruct_won_block_test nmc_mempool_name_test nmc_block_broadcast_test nmc_host_dualpath_test nmc_fallback_path_conformance_test dgb_gentx_share_path_test dgb_conn_pplns_producer_test dgb_other_tx_resolver_test \
                     dgb_other_tx_assembler_test dgb_reconstruct_won_block_test dgb_reconstruct_closure_test dgb_gentx_unpack_test dgb_work_source_test dgb_template_builder_test dgb_embedded_coin_node_test dgb_embedded_tx_select_test dgb_template_other_txs_test dgb_coinbase_value_parity_test dgb_submit_classify_test dgb_aux_parent_coinbase_parity_test dgb_template_capture_test dgb_aux_doge_db_commitment_bind_test dgb_aux_doge_mm_commitment_test dgb_aux_doge_dc_proof_test dgb_aux_doge_bind_parsers_test dgb_compact_blocks_bip152_parity_test dgb_aux_dual_target_select_test dgb_aux_broadcast_path_election_test dgb_aux_doge_submit_test dgb_aux_doge_embed_livewire_test dgb_aux_doge_dc_layout_verifier_test \
                     rpc_request_test softfork_check_test genesis_check_test algo_select_test digishield_walk_test header_chain_test \
                     dgb_coin_node_seam_test dgb_block_broadcast_test dgb_won_block_dispatch_test dgb_forced_won_share_dualpath_test dgb_scrypt_pow_test dgb_nonce_grinder_test dgb_regrind_block_test dgb_won_block_finalize_test dgb_share_target_genesis_test dgb_share_target_retarget_test dgb_share_bits_oracle_pin_test dgb_pool_msg_wire_test dgb_get_shares_walk_test dgb_download_stops_test dgb_think_p1_walk_bounds_test dgb_think_p1_desired_emit_test dgb_think_p6_desired_cutoff_test dgb_think_p4_head_keys_test dgb_think_p3_best_head_test dgb_g1_oracle_byte_parity_test dgb_think_p2_walk_bounds_test dgb_expected_time_to_block_test dgb_tail_score_endpoints_test dgb_pool_attempts_per_second_test dgb_pool_efficiency_test dgb_think_p5_best_share_punish_test dgb_auto_ratchet_tail_guard_test dgb_auto_ratchet_sim_test dgb_binomial_conf_interval_test dgb_desired_version_tally_test dgb_min_protocol_ratchet_test dgb_get_height_and_last_endpoints_test dgb_chain_walk_window_test dgb_redistribute_delegate_ghal_test dgb_share_weight_decay_test dgb_naughty_propagation_test dgb_hash_format_parity_test dgb_emergency_decay_saturation_test dgb_arith256_muldiv_kat_test test_coin_broadcaster test_multiaddress_pplns test_pplns_stress \

src/impl/nmc/coin/template_builder.hpp

@@ -131,6 +131,40 @@ inline std::string bits_to_hex(uint32_t bits) {
     return std::string(buf);
 }
 
+// Underfill guard (v36 cutover deploy path)
+//
+// Port of the LTC/DOGE template-builder guard (src/impl/ltc/coin/
+// template_builder.hpp, src/impl/doge/coin/template_builder.hpp) to the NMC
+// embedded template path, in the DASH free-predicate form
+// (src/impl/dash/coin/embedded_gbt.hpp) — same shape as the BTC port
+// (src/impl/btc/coin/template_builder.hpp). Detects the "near-empty template
+// on a non-empty mempool" regression: the tx selector returns almost no
+// transactions even though the local mempool holds a substantial fee-paying
+// backlog. c2pool-side template-fill safety net — NOT the byte-parity KAT
+// axis; thresholds are the v36-native shared structure (standardize
+// cross-coin) pinned to the legacy p2pool near-empty floor (~50 kB),
+// identical to LTC/DOGE/DASH/BTC. NMC is BTC's merge-mined aux child, so a
+// near-empty aux block does not waste parent PoW (severity LOW) — this port
+// completes the all-coin underfill matrix.
+inline constexpr uint64_t UNDERFILL_MIN_FILL_BYTES = 50'000ull; // < this = near-empty block
+inline constexpr uint64_t UNDERFILL_BACKLOG_SLACK  = 50'000ull; // unselected fee-paying material that should have filled it
+
+/// Pure trip predicate — the exact boolean the LTC/DOGE guards evaluate.
+/// Factored out so the KAT can pin it without a log scraper:
+///   near_empty  : template packed fewer bytes than the near-empty floor
+///   has_backlog : the mempool holds fee-paying material (known fees > 0)
+///                 well beyond what was selected (> selected + slack)
+/// Genuinely empty (or fee-unknown-only) mempools never trip.
+inline bool underfill_guard_trips(uint64_t selected_bytes,
+                                  uint64_t mempool_bytes,
+                                  uint64_t mempool_known_fees)
+{
+    const bool near_empty  = selected_bytes < UNDERFILL_MIN_FILL_BYTES;
+    const bool has_backlog = mempool_known_fees > 0
+                          && mempool_bytes > selected_bytes + UNDERFILL_BACKLOG_SLACK;
+    return near_empty && has_backlog;
+}
+
 // TemplateBuilder
 
 /// Builds an NMC block template from a validated HeaderChain and Mempool.
@@ -165,10 +199,16 @@ class TemplateBuilder {
 
     /// Build a WorkData template from the current chain tip + mempool.
     /// Returns std::nullopt if the chain has no tip yet (not synced to genesis).
+    /// underfill_tripped: optional underfill-guard observation seam. Defaults
+    /// to nullptr so every existing caller is byte-for-byte unchanged
+    /// (SAFE-ADDITIVE); the guard KAT passes a bool to pin the wiring without
+    /// a log scraper. The guard itself is log-only (WARNING), exactly like
+    /// LTC/DOGE/BTC — it never alters the template.
     static std::optional<rpc::WorkData> build_template(
         const HeaderChain& chain,
         const Mempool&     pool,
-        bool               is_testnet = false)
+        bool               is_testnet = false,
+        bool*              underfill_tripped = nullptr)
     {
         (void)is_testnet;  // reserved for future per-network rules
         auto t0 = std::chrono::steady_clock::now();
@@ -224,9 +264,11 @@ class TemplateBuilder {
         std::vector<Transaction> tx_objects;
         std::vector<uint256>     tx_hashes;
 
+        uint64_t selected_bytes = 0;  // wire bytes packed into this template (underfill guard)
         for (const auto& stx : selected_txs) {
             uint256     txid     = compute_txid(stx.tx);
             auto        packed   = pack(TX_WITH_WITNESS(stx.tx));
+            selected_bytes += packed.get_span().size();
             std::string hex_data = HexStr(packed.get_span());
             // wtxid = SHA256d of witness serialization (for witness merkle tree)
             uint256     wtxid    = Hash(packed.get_span());
@@ -245,6 +287,32 @@ class TemplateBuilder {
             tx_hashes.push_back(txid);
         }
 
+        // Underfill guard
+        //
+        // Do not silently treat a near-empty template as healthy when the
+        // mempool held fee-paying backlog that should have filled it. We cannot
+        // fabricate transactions, so we surface loudly (WARNING) for
+        // contabo-prod-watch / the operator rather than shipping a false-empty
+        // block as normal. Genuinely empty mempools never trip this. Mirrors
+        // the LTC/DOGE/BTC TemplateBuilder guard; additive only — the GBT JSON
+        // below is untouched either way.
+        {
+            const uint64_t mempool_bytes = static_cast<uint64_t>(pool.byte_size());
+            const uint64_t mempool_fees  = pool.total_fees();
+            const bool tripped = underfill_guard_trips(selected_bytes,
+                                                       mempool_bytes,
+                                                       mempool_fees);
+            if (underfill_tripped) *underfill_tripped = tripped;
+            if (tripped) {
+                LOG_WARNING << "[EMB-NMC] TemplateBuilder UNDERFILL: selected "
+                            << selected_txs.size() << " tx / " << selected_bytes
+                            << "B into template while mempool holds " << pool.size()
+                            << " tx / " << mempool_bytes << "B (" << mempool_fees
+                            << " sat fees) — near-empty aux block on a non-empty "
+                            << "mempool; template-fill regression, gates cutover.";
+            }
+        }
+
         // Build GBT-compatible JSON
         nlohmann::json data;
         data["version"]           = static_cast<int>(block_version);

src/impl/nmc/test/CMakeLists.txt

@@ -27,6 +27,8 @@ if (BUILD_TESTING AND GTest_FOUND)
     # P1 PC: embedded template builder KATs. Header-only builder
     # (template_builder.hpp / rpc_data.hpp); links the same SCC the sibling
     # merkle test does (core + nmc_coin for the out-of-line transaction ctor).
+    # The underfill guard KAT that used to be compiled into this target now
+    # lives in its own nmc_underfill_guard_test target below (btc/dash/dgb symmetry).
     add_executable(nmc_template_builder_test nmc_template_builder_test.cpp)
     target_link_libraries(nmc_template_builder_test PRIVATE
         GTest::gtest_main GTest::gtest
@@ -35,6 +37,24 @@ if (BUILD_TESTING AND GTest_FOUND)
         c2pool_payout c2pool_merged_mining c2pool_hashrate c2pool_storage)
     target_link_libraries(nmc_template_builder_test PRIVATE nmc_coin)
 
+    # nmc_underfill_guard_test: port of the LTC/DOGE "near-empty template on a
+    # non-empty mempool" template-builder guard to the NMC embedded template
+    # path (coin/template_builder.hpp), matching the BTC/DGB/DASH ports. Predicate
+    # KATs at the pinned 50 kB floor/slack + build_template wiring via the
+    # SAFE-ADDITIVE underfill_tripped seam (log-only guard; GBT projection asserted
+    # unchanged). Standalone target mirroring btc/dash/dgb *_underfill_guard_test
+    # for symmetry; same SCC as the sibling template_builder test (core + nmc_coin
+    # for the out-of-line transaction ctor). MUST be listed in the build.yml
+    # --target allowlist below (a registered target absent from that allowlist
+    # reds master as a NOT_BUILT sentinel: the #724/#728 lesson).
+    add_executable(nmc_underfill_guard_test nmc_underfill_guard_test.cpp)
+    target_link_libraries(nmc_underfill_guard_test PRIVATE
+        GTest::gtest_main GTest::gtest
+        core nlohmann_json::nlohmann_json)
+    target_link_libraries(nmc_underfill_guard_test PRIVATE
+        c2pool_payout c2pool_merged_mining c2pool_hashrate c2pool_storage)
+    target_link_libraries(nmc_underfill_guard_test PRIVATE nmc_coin)
+
     # P1 card 237: wire-vector KAT. Round-trips the canonical CAuxPow layout
     # against a REAL Namecoin mainnet block (757000). Same SCC as the sibling
     # tests (core + nmc_coin for the out-of-line MutableTransaction ctor).
@@ -110,6 +130,7 @@ if (BUILD_TESTING AND GTest_FOUND)
     include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR})
     gtest_add_tests(nmc_auxpow_merkle_test "" AUTO)
     gtest_add_tests(nmc_template_builder_test "" AUTO)
+    gtest_add_tests(nmc_underfill_guard_test "" AUTO)
     gtest_add_tests(nmc_auxpow_wire_test "" AUTO)
     gtest_add_tests(nmc_reconstruct_won_block_test "" AUTO)
     gtest_add_tests(nmc_mempool_name_test "" AUTO)