ltc: budget Phase-1 think() verify + compute-thread watchdog (.157 wedge fix)

src/impl/ltc/node.cpp

@@ -11,6 +11,9 @@
 #include <fstream>
 #include <iomanip>
 #include <random>
+#ifndef _WIN32
+#include <execinfo.h>  // backtrace() for think() watchdog stack dump (glibc-only)
+#endif
 
 // Static members for DensePPLNSWindow precomputed decay table
 std::vector<uint64_t> ltc::DensePPLNSWindow::s_decay_table;
@@ -1388,6 +1391,78 @@ void NodeImpl::prune_shares(const uint256& /*best_share*/)
 
 // (old phases 5-7 removed — replaced by p2pool-style pruning above)
 
+// ── think() watchdog (IO-thread timer; never touches m_tracker_mutex) ────
+// Atomic deadline + generation counter so a late fire cannot act on a newer
+// cycle. Mirrors btc/node.cpp. Armed on dispatch (run_think / clean_tracker),
+// disarmed in the corresponding IO-phase on normal completion.
+void NodeImpl::arm_think_watchdog()
+{
+    if (!m_context)
+        return;
+    auto deadline = std::chrono::steady_clock::now()
+                  + std::chrono::seconds(THINK_WATCHDOG_SECONDS);
+    m_think_deadline_ns.store(
+        std::chrono::duration_cast<std::chrono::nanoseconds>(
+            deadline.time_since_epoch()).count(),
+        std::memory_order_relaxed);
+    uint64_t gen = m_think_generation.fetch_add(1, std::memory_order_relaxed) + 1;
+
+    if (!m_watchdog_timer)
+        m_watchdog_timer = std::make_unique<boost::asio::steady_timer>(*m_context);
+    m_watchdog_timer->expires_after(std::chrono::seconds(THINK_WATCHDOG_SECONDS));
+    m_watchdog_timer->async_wait([this, gen](const boost::system::error_code& ec) {
+        if (ec) return;  // cancelled by disarm (normal completion)
+        if (m_think_generation.load(std::memory_order_relaxed) != gen)
+            return;  // a newer cycle armed since — not ours
+        int64_t dl = m_think_deadline_ns.load(std::memory_order_relaxed);
+        if (dl == 0)
+            return;  // cycle completed in the gap before this fire
+        auto now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
+            std::chrono::steady_clock::now().time_since_epoch()).count();
+        if (now_ns < dl)
+            return;  // not actually overdue (spurious) — let next arm handle it
+
+        LOG_ERROR << "[THINK-WATCHDOG] think()/clean cycle exceeded "
+                  << THINK_WATCHDOG_SECONDS << "s (gen=" << gen
+                  << ", pending_adds=" << m_pending_adds.size()
+                  << ") — compute thread appears wedged under the tracker lock;"
+                  << " recovering pipeline";
+#ifndef _WIN32
+        {
+            void* frames[64];
+            int n = ::backtrace(frames, 64);
+            char** syms = ::backtrace_symbols(frames, n);
+            if (syms) {
+                for (int i = 0; i < n; ++i)
+                    LOG_ERROR << "[THINK-WATCHDOG] bt[" << i << "] " << syms[i];
+                ::free(syms);
+            }
+        }
+#else
+        // Native backtrace is glibc-only (execinfo.h); on MSVC the timeout log
+        // above plus pending_adds is the diagnostic. Recovery below is identical.
+#endif
+
+        // Let a fresh cycle be scheduled. Does NOT unwind the stuck compute
+        // thread; the m_think_pool has a single thread, so a re-dispatched
+        // think() simply queues behind the wedged one (no concurrency), and the
+        // stuck cycle's IO-phase is idempotent if it ever returns.
+        m_think_deadline_ns.store(0, std::memory_order_relaxed);
+        m_think_running.store(false, std::memory_order_relaxed);
+        LOG_WARNING << "[THINK-WATCHDOG] m_think_running reset to false; "
+                       "a new think() cycle may now be scheduled";
+    });
+}
+
+void NodeImpl::disarm_think_watchdog()
+{
+    // Mark cycle complete: clears the deadline so a racing watchdog fire is a
+    // no-op, and cancels the pending timer.
+    m_think_deadline_ns.store(0, std::memory_order_relaxed);
+    if (m_watchdog_timer)
+        m_watchdog_timer->cancel();
+}
+
 void NodeImpl::run_think()
 {
     // Skip if a think() is already running on the compute thread.
@@ -1398,6 +1473,10 @@ void NodeImpl::run_think()
             LOG_INFO << "[ASYNC-THINK] skipped — compute thread busy (skip_count=" << skip_log << ")";
         return;
     }
+
+    // Arm the compute-thread watchdog for this think() cycle (disarmed in the
+    // IO-phase on normal completion).
+    arm_think_watchdog();
     LOG_INFO << "[ASYNC-THINK] dispatching to compute thread"
              << " pending_adds=" << m_pending_adds.size()
              << " peers=" << m_peers.size();
@@ -1498,6 +1577,9 @@ void NodeImpl::run_think()
       boost::asio::post(*m_context, [this, result = std::move(result),
                                       best_changed, needs_continue]() {
         try {
+            // think() returned normally — disarm the watchdog for this cycle.
+            disarm_think_watchdog();
+
             // Ban peers that provided invalid/unverifiable shares
             auto now = std::chrono::steady_clock::now();
             for (const auto& bad_addr : result.bad_peer_addresses) {
@@ -1795,6 +1877,10 @@ void NodeImpl::clean_tracker()
     // never concurrent with think() or IO-thread reads.
     m_think_running.store(true);  // block think() re-entry during clean
 
+    // clean_tracker runs think()+the removal walk under the same exclusive
+    // lock — arm the watchdog here too (disarmed in this path's IO-phase).
+    arm_think_watchdog();
+
     boost::asio::post(m_think_pool, [this]() {
       m_compute_thread_id.store(std::this_thread::get_id(), std::memory_order_relaxed);
 
@@ -2063,6 +2149,7 @@ void NodeImpl::clean_tracker()
       // Work refresh (1-5s) runs WITHOUT any lock so shared_lock readers
       // (handle_get_share, send_shares) are never blocked.
       boost::asio::post(*m_context, [this, clean_best_changed]() {
+        disarm_think_watchdog();
         if (clean_best_changed && m_on_best_share_changed) {
             LOG_INFO << "[CLEAN] IO-phase: work refresh (best changed)";
             m_on_best_share_changed();

src/impl/ltc/node.hpp

@@ -72,6 +72,26 @@ class NodeImpl : public pool::BaseNode<ltc::Config, ltc::ShareChain, ltc::Peer>
     std::atomic<bool> m_clean_running{false};
     mutable std::shared_mutex m_tracker_mutex;
 
+    // ── think()/clean watchdog (mirrors btc/node) ────────────────────────
+    // Best-effort recovery if a think()/clean cycle wedges while holding the
+    // exclusive tracker lock (the unbudgeted Phase-1 reorg-walk storm, .157
+    // 2026-06-19 19:53). The pre-existing io_context-liveness watchdog is BLIND
+    // to this: the IO thread only ever try_to_locks, so io_context stays
+    // responsive while the compute thread spins under the exclusive lock. This
+    // watchdog keys on a compute-thread deadline instead — an atomic ns-deadline
+    // polled by an IO-thread steady_timer that NEVER touches m_tracker_mutex
+    // (it would itself block on the stuck compute thread). On expiry it logs +
+    // dumps a backtrace, clears the deadline, and resets m_think_running so a
+    // fresh cycle can be scheduled. It does NOT forcibly unwind the wedged
+    // compute thread (unsafe); the stuck cycle, if it ever returns, finds the
+    // flag already false and its idempotent IO-phase drain runs harmlessly.
+    static constexpr int THINK_WATCHDOG_SECONDS = 30;
+    std::atomic<int64_t>  m_think_deadline_ns{0};
+    std::atomic<uint64_t> m_think_generation{0};
+    std::unique_ptr<boost::asio::steady_timer> m_watchdog_timer;
+    void arm_think_watchdog();
+    void disarm_think_watchdog();
+
     // ── Lock-free stats snapshot ─────────────────────────────────────────
     // Published by think() on the compute thread under m_tracker_mutex.
     // Read by ALL consumers (sync_status, loading page, global_stats,

src/impl/ltc/share_tracker.hpp

@@ -676,6 +676,26 @@
         // If verification fails: remove the share (it's bad).
         // If no verification possible and chain unrooted: request parents.
         std::vector<uint256> bads;
+
+        // Reset the cross-phase verification-continuation flag once per think()
+        // cycle. Both Phase 1 and Phase 2 set it when their budget is exhausted;
+        // run_think() then reposts a continuation that releases+reacquires the
+        // exclusive lock between chunks (the lock-segmentation seam).
+        m_think_needs_continue = false;
+
+        // Phase-1 verification budget — async-model adaptation mirroring the
+        // Phase-2 THINK_VERIFY_BUDGET below. p2pool walks walk_count=head_height
+        // for rooted heads (last is None, data.py:2131) but prunes aggressively
+        // so `last` is the prune boundary -> walk <=5. c2pool can hold a large
+        // rooted chain unpruned (last.IsNull(), head_height ~= chain.size()); a
+        // backward reorg that makes every head's tip fail attempt_verify then
+        // walks the FULL chain x PPLNS under the exclusive tracker lock ->
+        // multi-minute wedge (.157 2026-06-19 19:53). Budgeting bounds the
+        // per-cycle lock hold; the remainder verifies on the next run_think()
+        // continuation. The SET of shares verified is identical to p2pool --
+        // only the scheduling is chunked, so GENTX parity is preserved.
+        constexpr int THINK_P1_VERIFY_BUDGET = 100;
+        int p1_budget_remaining = bootstrap_mode ? INT_MAX : THINK_P1_VERIFY_BUDGET;
         {
             // Snapshot heads — we'll modify chain during iteration
             auto heads_snapshot = chain.get_heads();
@@ -688,6 +708,10 @@
             }
             for (auto& [head_hash, tail_hash] : heads_snapshot)
             {
+                // Phase-1 budget exhausted — defer the remaining heads to the
+                // next run_think() continuation (lock released between chunks).
+                if (p1_budget_remaining <= 0) { m_think_needs_continue = true; break; }
+
                 if (verified.get_heads().contains(head_hash)) {
                     ++p1_skipped;
                     continue;
@@ -744,7 +768,9 @@
                     auto chain_view = chain.get_chain(head_hash, walk_count);
                     for (auto [hash, data] : chain_view)
                     {
-                        if (attempt_verify(hash))
+                        bool verify_ok = attempt_verify(hash);
+                        --p1_budget_remaining;  // bound the per-cycle verify storm
+                        if (verify_ok)
                         {
                             verified_one = true;
                             ++p1_verified;
@@ -756,6 +782,9 @@
                         // false for mid-chain shares (NotImplementedError in
                         // p2pool), which is caught below.
                         bads.push_back(hash);
+                        // Budget spent mid-walk: stop here, resume next cycle so
+                        // the exclusive lock is released and IO can progress.
+                        if (p1_budget_remaining <= 0) { m_think_needs_continue = true; break; }
                     }
                 } catch (const std::exception& ex) {
                     ++p1_caught;
@@ -859,7 +888,9 @@
         // now, so budgeting is a safety net for cold starts only.
         constexpr int THINK_VERIFY_BUDGET = 100;
         int budget_remaining = bootstrap_mode ? INT_MAX : THINK_VERIFY_BUDGET;
-        m_think_needs_continue = false;
+        // NOTE: m_think_needs_continue is reset once at the top of think();
+        // Phase 1 may have already requested a continuation, so do NOT clear
+        // it here — either phase having remaining work must keep the loop going.
         {
             static int p2_skip_log = 0;
             if (p2_skip_log++ % 20 == 0)