Fix critical network stall and recovery deadlock issues

- Add time-based emergency recovery to prevent infinite deadlock
  * Introduce EMERGENCY_STOP_TIME for wall-clock tracking
  * Recovery now triggers after 10 seconds OR 10 blocks (whichever first)
  * Prevents nodes from being stuck forever when blocks stop

- Implement global network stall detection
  * Add LAST_BLOCK_PRODUCED_TIME/HEIGHT for tracking
  * Check every 30 seconds if no blocks produced for 10+ seconds
  * Automatically trigger emergency failover after 15 seconds

- Lower fast sync threshold from 50 to 10 blocks
  * Faster detection of node desynchronization
  * Prevents gradual divergence accumulation

- Improve fork detection and resolution
  * Immediate sync when own blocks rejected as forks
  * Bypass rate limiting for critical fork scenarios
  * Aggressive sync with network majority

Architecture preserved:
- Uses existing methods (select_emergency_producer, broadcast_emergency_producer_change)
- Lock-free atomic operations for scalability
- No additional network overhead in normal operation
- Supports Super/Full/Light node types correctly

Scalability for millions of nodes:
- O(1) atomic operations, no locks
- Local detection without network queries
- Emergency broadcasts only when needed
- No continuous heartbeat required

Author: AIQnetLab

development/qnet-integration/src/node.rs

@@ -15,7 +15,7 @@ pub const MIN_COMPATIBLE_VERSION: u32 = 1;  // Minimum version we can work with
 // PRODUCTION CONSTANTS - No hardcoded magic numbers!
 const ROTATION_INTERVAL_BLOCKS: u64 = 30; // Producer rotation every 30 blocks
 const MIN_BYZANTINE_NODES: usize = 4; // 3f+1 where f=1
-const FAST_SYNC_THRESHOLD: u64 = 50; // Trigger fast sync if behind by 50+ blocks  
+const FAST_SYNC_THRESHOLD: u64 = 10; // Trigger fast sync if behind by 10+ blocks (lowered from 50 for faster detection)  
 const FAST_SYNC_TIMEOUT_SECS: u64 = 60; // Fast sync timeout
 const BACKGROUND_SYNC_TIMEOUT_SECS: u64 = 30; // Background sync timeout
 const SNAPSHOT_FULL_INTERVAL: u64 = 10000; // Full snapshot every 10k blocks
@@ -69,11 +69,16 @@ pub fn set_emergency_producer_flag(block_height: u64, producer: String) {
 }
 
 // CRITICAL: Global synchronization flags for API access
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 static SYNC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
 static FAST_SYNC_IN_PROGRESS: AtomicBool = AtomicBool::new(false);
 pub static NODE_IS_SYNCHRONIZED: AtomicBool = AtomicBool::new(false);
 
+// CRITICAL FIX: Track last block production time globally for stall detection
+// This prevents network from getting stuck when all nodes stop producing
+static LAST_BLOCK_PRODUCED_TIME: AtomicU64 = AtomicU64::new(0);
+static LAST_BLOCK_PRODUCED_HEIGHT: AtomicU64 = AtomicU64::new(0);
+
 // NOTE: Removed ROTATION_NOTIFY - simple 1-second timing is more reliable
 // Testing showed that natural timing without interrupts prevents race conditions
 
@@ -1444,9 +1449,14 @@ impl BlockchainNode {
                                             continue;
                                         }
 
-                                        // CRITICAL: Rate limit fork attempts (prevent DoS)
+                                        // CRITICAL: Rate limit fork attempts (prevent DoS) 
+                                        // But allow immediate sync if it's our own fork
                                         let last_attempt = *last_fork_attempt.read().await;
-                                        if last_attempt.elapsed().as_secs() < FORK_ATTEMPT_COOLDOWN_SECS {
+                                        let own_fork = if let Some(p2p) = &unified_p2p {
+                                            fork_producer == p2p.get_node_id()
+                                        } else { false };
+                                        
+                                        if !own_fork && last_attempt.elapsed().as_secs() < FORK_ATTEMPT_COOLDOWN_SECS {
                                             println!("[REORG] 🛡️ Fork attempt too soon - rate limited ({}s cooldown)", FORK_ATTEMPT_COOLDOWN_SECS);
 
                                             // CRITICAL FIX: If we detect our own blocks being rejected as forks,
@@ -1689,6 +1699,10 @@ impl BlockchainNode {
                             *height.write().await = received_block.height;
                             println!("[BLOCKS] 📊 Global height updated to {}", received_block.height);
 
+                            // CRITICAL FIX: Update last block time for stall detection
+                            LAST_BLOCK_PRODUCED_TIME.store(get_timestamp_safe(), Ordering::Relaxed);
+                            LAST_BLOCK_PRODUCED_HEIGHT.store(received_block.height, Ordering::Relaxed);
+                            
                             // CRITICAL FIX: Update P2P local height for message filtering
                             crate::unified_p2p::LOCAL_BLOCKCHAIN_HEIGHT.store(
                                 received_block.height, 
@@ -2875,6 +2889,50 @@ impl BlockchainNode {
                         last_production_height = microblock_height;
                         last_production_time = std::time::Instant::now();
                     }
+                    
+                    // CRITICAL FIX: Global network stall detection
+                    // Check if ANY block has been produced recently (by us or others)
+                    let last_block_time = LAST_BLOCK_PRODUCED_TIME.load(Ordering::Relaxed);
+                    let last_block_height = LAST_BLOCK_PRODUCED_HEIGHT.load(Ordering::Relaxed);
+                    let current_time = get_timestamp_safe();
+                    
+                    if last_block_time > 0 && current_time > last_block_time {
+                        let time_since_last_block = current_time - last_block_time;
+                        
+                        // CRITICAL: Trigger emergency if no blocks for 10+ seconds
+                        // This is GLOBAL stall detection, not just local
+                        if time_since_last_block > 10 && microblock_height > 0 {
+                            println!("[STALL] 🚨 NETWORK STALL DETECTED! No blocks for {} seconds", time_since_last_block);
+                            println!("[STALL] 📊 Last block: #{} at timestamp {}", last_block_height, last_block_time);
+                            
+                            // Force emergency producer selection if we're supposed to be producing
+                            if let Some(p2p) = &unified_p2p {
+                                let next_height = microblock_height + 1;
+                                let expected_producer = Self::select_microblock_producer(
+                                    next_height, &unified_p2p, &node_id, node_type,
+                                    Some(&storage), &quantum_poh
+                                ).await;
+                                
+                                if time_since_last_block > 15 {
+                                    println!("[STALL] 🔥 Triggering emergency failover for producer: {}", expected_producer);
+                                    
+                                    // Select emergency producer
+                                    let emergency_producer = Self::select_emergency_producer(
+                                        &expected_producer, next_height, &unified_p2p,
+                                        &node_id, node_type, Some(storage.clone())
+                                    ).await;
+                                    
+                                    // Broadcast emergency change
+                                    if let Err(e) = p2p.broadcast_emergency_producer_change(
+                                        &expected_producer, &emergency_producer, 
+                                        next_height, "network_stall"
+                                    ) {
+                                        println!("[STALL] ⚠️ Failed to broadcast emergency: {}", e);
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
                 // SYNC FIX: Fast catch-up mode for nodes that are far behind
                 // Using global flags defined at module level
@@ -2936,8 +2994,8 @@ impl BlockchainNode {
                             println!("[SYNC] ⚠️ Node is {} blocks behind network (local: {}, network: {})", 
                                      height_difference, microblock_height, network_height);
 
-                            // For extreme lag (>50 blocks), use fast sync
-                        if height_difference > 50 {
+                            // For significant lag (>10 blocks), use fast sync
+                        if height_difference > 10 {
                             // RACE CONDITION FIX: Only start fast sync if not already running
                                 if !FAST_SYNC_IN_PROGRESS.swap(true, Ordering::SeqCst) {
                                 println!("[SYNC] ⚡ FAST SYNC MODE: {} blocks behind, catching up...", height_difference);
@@ -4049,6 +4107,10 @@ impl BlockchainNode {
                     // This prevents phantom height where node claims height N without having block N
                     println!("[PRODUCER] ✅ Created and saved block #{}", microblock.height);
 
+                    // CRITICAL FIX: Update global last block time for stall detection
+                    LAST_BLOCK_PRODUCED_TIME.store(get_timestamp_safe(), Ordering::Relaxed);
+                    LAST_BLOCK_PRODUCED_HEIGHT.store(microblock.height, Ordering::Relaxed);
+                    
                     // CRITICAL: Increment height for next iteration
                     // We only advance after successfully creating and storing the block
                     microblock_height = microblock.height;  // Set to the block we just created

development/qnet-integration/src/unified_p2p.rs

@@ -101,6 +101,11 @@ pub static EMERGENCY_STOP_PRODUCTION: Lazy<Arc<AtomicBool>> =
 pub static EMERGENCY_STOP_HEIGHT: Lazy<Arc<AtomicU64>> = 
     Lazy::new(|| Arc::new(AtomicU64::new(0)));
 
+// CRITICAL FIX: Track TIME of emergency stop to prevent deadlock
+// Recovery after 10 seconds (not blocks) to avoid infinite wait
+pub static EMERGENCY_STOP_TIME: Lazy<Arc<AtomicU64>> = 
+    Lazy::new(|| Arc::new(AtomicU64::new(0)));
+
 // CRITICAL: Track emergency failovers in progress to prevent race conditions
 // Format: "emergency_failover_{height}" -> prevents multiple nodes from initiating same failover
 // SCALABILITY: DashSet for lock-free concurrent access with millions of nodes
@@ -7753,7 +7758,13 @@ impl SimplifiedP2P {
                     let current_stop_height = EMERGENCY_STOP_HEIGHT.load(Ordering::Relaxed);
                     if current_stop_height == 0 {
                         EMERGENCY_STOP_HEIGHT.store(block_height, Ordering::Relaxed);
-                        println!("[RECOVERY] 📍 Will auto-recover after 10 blocks (at block #{})", block_height + 10);
+                        // CRITICAL FIX: Also store TIME to prevent deadlock when blocks stop
+                        let current_time = std::time::SystemTime::now()
+                            .duration_since(std::time::UNIX_EPOCH)
+                            .unwrap_or_default()
+                            .as_secs();
+                        EMERGENCY_STOP_TIME.store(current_time, Ordering::Relaxed);
+                        println!("[RECOVERY] 📍 Will auto-recover after 10 seconds (time-based) or 10 blocks");
                     } else {
                         println!("[RECOVERY] ⚠️ Already stopped at block #{}, not resetting timer", current_stop_height);
                     }
@@ -7767,19 +7778,33 @@ impl SimplifiedP2P {
             }
         }
 
-        // Check if we should clear the emergency stop (been stopped for 10+ blocks)
+        // Check if we should clear the emergency stop (been stopped for 10+ blocks OR 10+ seconds)
         // This applies to Super/Full nodes that were previously stopped
         if EMERGENCY_STOP_PRODUCTION.load(Ordering::Relaxed) {
             let stop_height = EMERGENCY_STOP_HEIGHT.load(Ordering::Relaxed);
-            if stop_height > 0 && block_height >= stop_height + 10 {
-                println!("[RECOVERY] ✅ Auto-clearing emergency stop after 10 blocks (stopped at #{}, now at #{})", 
-                        stop_height, block_height);
+            let stop_time = EMERGENCY_STOP_TIME.load(Ordering::Relaxed);
+            let current_time = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap_or_default()
+                .as_secs();
+            
+            // CRITICAL FIX: Clear stop EITHER after 10 blocks OR 10 seconds (whichever comes first)
+            // This prevents deadlock when network stops producing blocks
+            let blocks_passed = if block_height > stop_height { block_height - stop_height } else { 0 };
+            let seconds_passed = if current_time > stop_time { current_time - stop_time } else { 0 };
+            
+            if stop_height > 0 && (blocks_passed >= 10 || seconds_passed >= 10) {
+                println!("[RECOVERY] ✅ Auto-clearing emergency stop after {} blocks / {} seconds", 
+                        blocks_passed, seconds_passed);
                 EMERGENCY_STOP_PRODUCTION.store(false, Ordering::Relaxed);
                 EMERGENCY_STOP_HEIGHT.store(0, Ordering::Relaxed);
+                EMERGENCY_STOP_TIME.store(0, Ordering::Relaxed);
                 println!("[RECOVERY] 🚀 Node can now resume block production");
             } else if stop_height > 0 {
-                let blocks_remaining = 10 - (block_height - stop_height);
-                println!("[RECOVERY] ⏳ Emergency stop active for {} more blocks", blocks_remaining);
+                let blocks_remaining = 10_u64.saturating_sub(blocks_passed);
+                let seconds_remaining = 10_u64.saturating_sub(seconds_passed);
+                println!("[RECOVERY] ⏳ Emergency stop active for {} more blocks OR {} more seconds", 
+                        blocks_remaining, seconds_remaining);
             }
         }