Skip to content

Commit 4995cf1

Browse files
committed
Fix network stall during macroblock consensus
- Increased timeout to 15s during consensus period (blocks 61-90) to prevent false emergency detection - Added active emergency recovery check in main loop (every 1 second) - Recovery now triggers after 10 seconds OR 10 blocks (whichever comes first) - Invalidate producer cache on emergency change and recovery - Added LAST_BLOCK_PRODUCED_TIME and LAST_BLOCK_PRODUCED_HEIGHT for global stall detection - Lowered FAST_SYNC_THRESHOLD from 50 to 10 blocks for faster sync - Improved fork detection to allow immediate sync when own blocks are rejected This prevents network deadlocks where: 1. Producer experiences delays during consensus overhead 2. Other nodes incorrectly trigger emergency failover 3. Failed producer gets stuck in emergency stop state 4. Network stalls because no blocks are produced The fix ensures microblocks continue at 1 block/second while macroblock consensus runs in parallel.
1 parent 583b484 commit 4995cf1

2 files changed

Lines changed: 41 additions & 1 deletion

File tree

development/qnet-integration/src/node.rs

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2865,6 +2865,36 @@ impl BlockchainNode {
28652865
}
28662866
}
28672867

2868+
// CRITICAL FIX: Check emergency recovery EVERY SECOND, not just on messages
2869+
// This prevents deadlock when network stops and no messages arrive
2870+
if crate::unified_p2p::EMERGENCY_STOP_PRODUCTION.load(Ordering::Relaxed) {
2871+
let stop_height = crate::unified_p2p::EMERGENCY_STOP_HEIGHT.load(Ordering::Relaxed);
2872+
let stop_time = crate::unified_p2p::EMERGENCY_STOP_TIME.load(Ordering::Relaxed);
2873+
let current_time = get_timestamp_safe();
2874+
2875+
if stop_height > 0 && stop_time > 0 {
2876+
let blocks_passed = if microblock_height > stop_height {
2877+
microblock_height - stop_height
2878+
} else { 0 };
2879+
let seconds_passed = if current_time > stop_time {
2880+
current_time - stop_time
2881+
} else { 0 };
2882+
2883+
// Clear emergency stop after 10 blocks OR 10 seconds
2884+
if blocks_passed >= 10 || seconds_passed >= 10 {
2885+
println!("[RECOVERY] ✅ Auto-clearing emergency stop in main loop ({}s / {} blocks passed)",
2886+
seconds_passed, blocks_passed);
2887+
crate::unified_p2p::EMERGENCY_STOP_PRODUCTION.store(false, Ordering::Relaxed);
2888+
crate::unified_p2p::EMERGENCY_STOP_HEIGHT.store(0, Ordering::Relaxed);
2889+
crate::unified_p2p::EMERGENCY_STOP_TIME.store(0, Ordering::Relaxed);
2890+
2891+
// CRITICAL: Invalidate producer cache to allow this node to be selected again
2892+
Self::invalidate_producer_cache();
2893+
println!("[RECOVERY] 🚀 Node can now resume block production");
2894+
}
2895+
}
2896+
}
2897+
28682898
// CPU OPTIMIZATION: Log CPU stats every 30 seconds
28692899
if cpu_check_counter % 30 == 0 {
28702900
let elapsed = start_time.elapsed().as_secs();
@@ -4447,12 +4477,20 @@ impl BlockchainNode {
44474477
if !block_exists {
44484478
// CRITICAL FIX: Adaptive timeout based on network conditions
44494479
// Synchronous broadcast should arrive faster, but network delays still exist
4480+
4481+
// CRITICAL FIX: During macroblock consensus, allow more time
4482+
// Consensus runs in background but can affect block production timing
4483+
let blocks_since_last_macro = expected_height_timeout % 90;
4484+
let is_consensus_period = blocks_since_last_macro >= 61 && blocks_since_last_macro <= 90;
4485+
44504486
let timeout_duration = if expected_height_timeout == 1 {
44514487
20 // First block needs more time for network stabilization
44524488
} else if expected_height_timeout <= 10 {
44534489
10 // Early blocks: 10 seconds for initial sync
4490+
} else if is_consensus_period {
4491+
15 // During consensus: more tolerance (was causing false emergencies)
44544492
} else {
4455-
7 // Normal operation: 7 seconds (was 5, too aggressive)
4493+
7 // Normal operation: 7 seconds
44564494
};
44574495

44584496
// Special logging for rotation boundaries

development/qnet-integration/src/unified_p2p.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7885,6 +7885,8 @@ impl SimplifiedP2P {
78857885

78867886
// CRITICAL FIX: Invalidate producer cache to prevent selecting failed producer again
78877887
// This ensures the network will select a new producer in the next round
7888+
// IMPORTANT: Do this for ALL nodes, not just the emergency producer
7889+
println!("[FAILOVER] 🔄 Invalidating producer cache after emergency change");
78887890
crate::node::BlockchainNode::invalidate_producer_cache();
78897891
}
78907892

0 commit comments

Comments
 (0)