Skip to content

Commit 18f33e1

Browse files
AIQnetLabclaude
andcommitted
fix(consensus,cold-join): deterministic producer-eligibility re-entry for returning nodes
create_eligible_producers_snapshot built its registered-super set from a 14400-block microblock body scan, so a node registered before that window (genesis = block 0; any super past the L2 carryover) synced after a wipe but never re-entered the produce/ committee set. The set also feeds the QC-bound epoch_commitment yet was read from live, async-lagging state, so it could diverge across committee members (stall/fork vector). - Phase-1: source registered supers from the snapshot-carried srtr_ registry index (super_registrations_sorted), not a recent-block scan — a returning node re-enters via the Phase-2A heartbeat gate without re-registration. Same source the reward roster uses. - Bound each re-entry candidate to reg_height <= end_height (node_reg_height) so an ahead-of-end_height registration cannot enter the set on a faster member only. - recent_heartbeat_senders: derive Phase-2A recent-heartbeat liveness from committed microblock bodies bounded to end_height instead of load_account (the async best-effort accounts CF). Pure function of the canonical chain <= end_height, identical on every committee member — no live-tip divergence. - verify_snapshot_consensus_binding: deterministically wait for the co-located GALC capsule before selecting walk_root (early-exit when a held capsule is above the anchor; fall back to ws_floor on timeout). eligible_producers / epoch_commitment are now a pure function of state <= end_height. Unit test recency_subwindow_indices_boundary; 185 qnet-integration + 62 qnet-state lib tests pass; 0 warnings. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 1845fa1 commit 18f33e1

2 files changed

Lines changed: 139 additions & 27 deletions

File tree

development/qnet-integration/src/node.rs

Lines changed: 87 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -863,6 +863,46 @@ pub fn window_content_from_accum(mb_idx: u64) -> Option<(Vec<[u8; 32]>, Vec<[u8;
863863
Some((mb_hashes, vrf_outputs))
864864
}
865865

866+
/// v36: deterministic recent-Heartbeat liveness for Phase-2A producer eligibility.
867+
///
868+
/// Phase-2A previously read the recent-Heartbeat bit from `load_account(reg).heartbeat_slots`, i.e. the
869+
/// persisted `accounts` CF — which is written by a DETACHED best-effort persist (microblocks are the
870+
/// authoritative store). The eligible snapshot runs async, so each committee member read a different
871+
/// persist-lag prefix → divergent eligible_producers → the QC-bound epoch_commitment split → 2f+1 never
872+
/// formed → finality stall. Fix: derive the set from the COMMITTED block bodies (synchronously saved at
873+
/// apply, canonical + identical on every node), bounded to `scan_end`. Returns the supers that sent a
874+
/// Heartbeat whose ANCHOR fell in the current or previous subwindow (same epoch — mirrors the old bitmask
875+
/// recency) and that was included in a block at-or-below scan_end. A pure function of the canonical chain
876+
/// ≤ scan_end ⇒ identical on every committee member, with NO live-tip dependence. Scans ≤2 subwindows
877+
/// (~2880 blocks), off the production path; bodies are retained 6 epochs, far beyond this window.
878+
/// (current, previous) global subwindow indices (anchor/1440 = epoch*10 + subwindow) for `scan_end`. The
879+
/// previous counts ONLY within the SAME epoch — at a subwindow-0/epoch boundary it returns current==prev
880+
/// (no previous), mirroring the prior `heartbeat_epoch==hb_epoch` + `if cur_sub>0` bitmask recency. Pure
881+
/// ⇒ unit-tested for the off-by-one + cross-epoch boundary.
882+
fn recency_subwindow_indices(scan_end: u64) -> (u64, u64) {
883+
let cur_idx = scan_end / 1440;
884+
let cur_sub = (scan_end % 14400) / 1440; // 0..9 within the epoch
885+
let prev_idx = if cur_sub > 0 { cur_idx.saturating_sub(1) } else { cur_idx };
886+
(cur_idx, prev_idx)
887+
}
888+
889+
fn recent_heartbeat_senders(storage: &crate::storage::Storage, scan_end: u64) -> std::collections::HashSet<String> {
890+
let (cur_idx, prev_idx) = recency_subwindow_indices(scan_end);
891+
let start = prev_idx.saturating_mul(1440);
892+
let mut set: std::collections::HashSet<String> = std::collections::HashSet::new();
893+
for h in start..=scan_end {
894+
if let Ok(Some(block)) = storage.load_microblock_auto_format(h) {
895+
for tx in &block.transactions {
896+
if let qnet_state::TransactionType::Heartbeat { node_id, anchor_height, .. } = &tx.tx_type {
897+
let s = anchor_height / 1440;
898+
if s == cur_idx || s == prev_idx { set.insert(node_id.clone()); }
899+
}
900+
}
901+
}
902+
}
903+
set
904+
}
905+
866906
// ═══════════════════════════════════════════════════════════════════════════════
867907
// PRODUCTION v2.50: Lock-free global storage with OnceCell + Arc
868908
// RocksDB does NOT support multiple connections - single instance shared immutably
@@ -4098,24 +4138,22 @@ impl BlockchainNode {
40984138
// returning node's next HBC is always in range. Phase 1 builds the
40994139
// registered-super-node set; Phase 2A is the single eligibility path.
41004140
let scan_end = macroblock_index * 90;
4101-
const PHASE_2A_SCAN_BLOCKS: u64 = 14_400;
4102-
let scan_start = scan_end.saturating_sub(PHASE_2A_SCAN_BLOCKS);
4103-
4104-
// Phase 1: registered Super node IDs (necessary, not sufficient).
4105-
let mut registered_super_nodes: std::collections::HashSet<String> = std::collections::HashSet::new();
4106-
for height in scan_start..=scan_end {
4107-
if let Ok(Some(block)) = storage.load_microblock_auto_format(height) {
4108-
for tx in &block.transactions {
4109-
if let qnet_state::TransactionType::NodeRegistration {
4110-
node_id, node_type, ..
4111-
} = &tx.tx_type {
4112-
if *node_type == qnet_state::NodeType::Super {
4113-
registered_super_nodes.insert(node_id.clone());
4114-
}
4115-
}
4116-
}
4117-
}
4118-
}
4141+
4142+
// Phase 1: registered Super node IDs (necessary, not sufficient). Sourced from the
4143+
// deterministic, snapshot-carried srtr_ registry index — NOT a recent-block body scan.
4144+
// A body scan only saw registrations inside the last epoch, so a node whose NodeRegistration
4145+
// is older than that window (every genesis node: block 0; any super away beyond the L2
4146+
// carryover) could never re-enter the producer/committee set after a snapshot cold-join —
4147+
// it synced but stayed ineligible. The registry index holds EVERY chain-confirmed super
4148+
// regardless of registration age (the same source the reward roster uses), so a returning
4149+
// node re-enters through the Phase-2A heartbeat gate below WITHOUT re-registration. Phase-2A
4150+
// (recent on-chain Heartbeat) absorbs any registration-timing edge: a just-applied
4151+
// registration not yet heartbeated is filtered out, so eligible-set membership stays stable.
4152+
let registered_super_nodes: std::collections::HashSet<String> =
4153+
match storage.super_registrations_sorted() {
4154+
Ok(regs) => regs.into_iter().map(|(node_id, _w)| node_id).collect(),
4155+
Err(_) => std::collections::HashSet::new(),
4156+
};
41194157

41204158
// v35: Phase-2A admits a registered Super node on UNFORGEABLE on-chain liveness —
41214159
// a Heartbeat-TX in the current or previous subwindow (Account.heartbeat_slots),
@@ -4124,18 +4162,25 @@ impl BlockchainNode {
41244162
{
41254163
let hb_epoch = scan_end / 14400;
41264164
let cur_sub = ((scan_end % 14400) / 1440) as u16;
4165+
// Deterministic recent-Heartbeat set from committed block bodies bounded to end_height
4166+
// (NOT the async-lagging accounts CF, whose per-node persist lag gave a divergent eligible
4167+
// set → epoch_commitment split → finality stall). Computed once; identical on every member.
4168+
let recent_hb = recent_heartbeat_senders(storage, scan_end);
41274169
let mut regs: Vec<&String> = registered_super_nodes.iter().collect();
41284170
regs.sort();
41294171
let mut added_tally = 0usize;
41304172
for reg in regs {
41314173
if eligible_ids.contains(reg) { continue; }
4132-
let acct = match storage.load_account(reg).ok().flatten() {
4133-
Some(a) if a.heartbeat_epoch == hb_epoch => a,
4174+
// Determinism: the srtr_ candidate pool is read at the live applied tip, which under
4175+
// async production can run ahead of end_height; bound each re-entry candidate to a
4176+
// registration CONFIRMED by end_height so every committee member admits the SAME set
4177+
// (an ahead-of-end_height registration is excluded identically everywhere). Genesis
4178+
// nodes carry reg_height=0 ⇒ always pass.
4179+
match storage.node_reg_height(reg) {
4180+
Ok(Some(h)) if h <= scan_end => {}
41344181
_ => continue,
4135-
};
4136-
let mut recent = acct.heartbeat_slots & (1u16 << cur_sub.min(9));
4137-
if cur_sub > 0 { recent |= acct.heartbeat_slots & (1u16 << (cur_sub - 1)); }
4138-
if recent == 0 { continue; }
4182+
}
4183+
if !recent_hb.contains(reg) { continue; }
41394184
let rep = (reputation_map.get(reg).copied()
41404185
.unwrap_or(qnet_consensus::deterministic_reputation::INITIAL_REPUTATION)
41414186
.clamp(0.0, 100.0) * 100.0).round() as u32;
@@ -27732,6 +27777,24 @@ mod tests {
2773227777
assert!(!checkpoint_participation_allowed(false, 0, mb_end)); // syncing, no window → defer
2773327778
}
2773427779

27780+
// Phase-2A recency window (deterministic heartbeat eligibility): current subwindow + previous ONLY
27781+
// within the same epoch. The epoch boundary (subwindow 0) must NOT bridge to the prior epoch's
27782+
// subwindow 9 — that is the off-by-one that would change WHO is eligible vs the old bitmask gate.
27783+
#[test]
27784+
fn recency_subwindow_indices_boundary() {
27785+
// Mid-epoch: previous = current-1 (same epoch).
27786+
assert_eq!(recency_subwindow_indices(5 * 1440), (5, 4));
27787+
assert_eq!(recency_subwindow_indices(5 * 1440 + 100), (5, 4));
27788+
// Epoch start (subwindow 0): no previous within the epoch ⇒ prev == cur (degenerates to {cur}).
27789+
assert_eq!(recency_subwindow_indices(0), (0, 0));
27790+
assert_eq!(recency_subwindow_indices(14400), (10, 10)); // epoch1 sub0 ⇒ (10,10), NOT (10,9)
27791+
assert_eq!(recency_subwindow_indices(14400 + 50), (10, 10));
27792+
// Epoch1 subwindow 1: previous is sub0 of the SAME epoch (10), never the prior epoch's sub9 (9).
27793+
assert_eq!(recency_subwindow_indices(14400 + 1440), (11, 10));
27794+
// Epoch2 subwindow 0: again no bridge.
27795+
assert_eq!(recency_subwindow_indices(2 * 14400), (20, 20));
27796+
}
27797+
2773527798
// committee_for_height determinism: genesis era ⇒ None (caller uses the genesis committee), and
2773627799
// an ABSENT N-2 snapshot ⇒ None — REJECT, never a per-node walk-back guess (which would fork).
2773727800
#[test]

development/qnet-integration/src/storage.rs

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7016,7 +7016,26 @@ impl Storage {
70167016
None => Ok(None),
70177017
}
70187018
}
7019-
7019+
7020+
/// Chain-confirmed registration height of a node (None if unregistered or reg_height unstamped).
7021+
/// Used to bound the eligible-producer candidate set to registrations confirmed AS OF a macroblock
7022+
/// end_height: committee members at divergent live applied tips (production never waits for
7023+
/// consensus) must compute the SAME set, so an ahead-of-end_height registration must be excluded
7024+
/// identically on every node. Genesis nodes carry reg_height=0.
7025+
pub fn node_reg_height(&self, node_id: &str) -> IntegrationResult<Option<u64>> {
7026+
let registry_cf = self.persistent.db.cf_handle("node_registry")
7027+
.ok_or_else(|| IntegrationError::StorageError("node_registry column family not found".to_string()))?;
7028+
let key = format!("node_{}", node_id);
7029+
match self.persistent.db.get_cf(&registry_cf, key.as_bytes())? {
7030+
Some(data) => {
7031+
let parsed: serde_json::Value = serde_json::from_slice(&data)
7032+
.map_err(|e| IntegrationError::DeserializationError(e.to_string()))?;
7033+
Ok(parsed["reg_height"].as_u64())
7034+
}
7035+
None => Ok(None),
7036+
}
7037+
}
7038+
70207039
/// v4.3: Get all nodes registered with a specific wallet address — O(1) via reverse index
70217040
/// CRITICAL for mobile app: Returns nodes even when the node itself is offline!
70227041
/// Data is read from blockchain storage (RocksDB), not from the node's memory.
@@ -9743,15 +9762,45 @@ impl Storage {
97439762
// capsule is a walk SHORTENER, never a floor: a capsule ABOVE the anchor can't root the forward
97449763
// N-2 lineage walk DOWN to it, so it roots the walk ONLY when at-or-below the anchor; else ws_floor.
97459764
let ws_floor = crate::node::effective_ws_checkpoint();
9746-
let pin = crate::galc::effective_pin_checkpoint();
97479765
if mb_idx < ws_floor.0 {
97489766
let _ = self.discard_snapshot_state(snapshot_height);
97499767
return Err(IntegrationError::Other(format!(
97509768
"snapshot_below_ws mb={} ws={} action=reject_snapshot", mb_idx, ws_floor.0
97519769
)));
97529770
}
9771+
// Root the walk at the genesis-signed GALC capsule when one is co-located at/below the
9772+
// snapshot anchor (walk ≈ 0). The capsule arrives + Dilithium-verifies asynchronously, so a
9773+
// binding that ran right after the cold-join orchestrator's best-effort request would race it
9774+
// and fall back to ws_floor → a full genesis-to-anchor re-verify (the slow-rejoin bug).
9775+
// Deterministically request + bounded-wait for a usable capsule before rooting; on timeout
9776+
// fall through to ws_floor (correct, only slower — never worse, no new launch requirement).
9777+
let usable = |k: u64| k > ws_floor.0 && k <= mb_idx;
9778+
let mut pin = crate::galc::effective_pin_checkpoint();
9779+
if !usable(pin.0) {
9780+
const GALC_WAIT_ATTEMPTS: u32 = 20; // ≤ ~10s total
9781+
const GALC_WAIT_INTERVAL_MS: u64 = 500;
9782+
for i in 0..GALC_WAIT_ATTEMPTS {
9783+
// A capsule already adopted ABOVE the anchor (cadence put the freshest mint a step ahead
9784+
// of the negotiated snapshot) can never become usable by waiting — adoption is monotonic-up
9785+
// — so root at ws_floor now instead of burning the timeout; the next snapshot boundary
9786+
// re-aligns capsule and anchor.
9787+
if pin.0 > mb_idx { break; }
9788+
if i % 4 == 0 { // re-request every ~2s (a reply may be lost)
9789+
let _ = p2p.broadcast_quic(&crate::unified_p2p::NetworkMessage::RequestGenesisCheckpoint {
9790+
requester_id: "snapshot_binder".to_string(),
9791+
}).await;
9792+
}
9793+
tokio::time::sleep(std::time::Duration::from_millis(GALC_WAIT_INTERVAL_MS)).await;
9794+
pin = crate::galc::effective_pin_checkpoint();
9795+
if usable(pin.0) { break; }
9796+
}
9797+
if crate::node::is_info() {
9798+
println!("[INFO][SYNC] galc_anchor_wait mb={} pin={} rooted={}",
9799+
mb_idx, pin.0, if usable(pin.0) { "capsule" } else { "ws_floor" });
9800+
}
9801+
}
97539802
let walk_root: (u64, [u8; 32]) =
9754-
if pin.0 > ws_floor.0 && pin.0 <= mb_idx { (pin.0, pin.1) } else { ws_floor };
9803+
if usable(pin.0) { (pin.0, pin.1) } else { ws_floor };
97559804
// Bound the walk so a stale root can't degrade into an unbounded genesis-to-tip re-verify
97569805
// (DoS-on-self CPU + a wider trust window). The GALC capsule normally keeps the root within a
97579806
// few macroblocks of the anchor (walk ≈ 0); this is the FALLBACK ceiling when no capsule is

0 commit comments

Comments
 (0)