Skip to content

Commit f363368

Browse files
committed
Add enhanced logging and rollback detection
Node startup logging improvements: - Log data directory path on state load - Log current_round when loading DAG - Log last_finalized_round when loading finality tracker - Log total_supply and last_finalized_round when loading state engine - Log transaction count when loading mempool - Log when starting fresh (no state files found) This will help diagnose restart issues by showing exactly what state was loaded and from where. Extended monitoring enhancements: - Track previous round and supply across samples - Detect round rollbacks (current < previous) - Detect supply drops (>100k UDAG decrease) - Alert with 🚨 when rollback or supply drop detected - Output NODE:ROUND and NODE:SUPPLY for tracking This will immediately alert if the network restarts from old state, preventing undetected rollbacks like the one observed at 12:30:00. Critical for mainnet safety - rollback detection is essential.
1 parent d1b0c48 commit f363368

2 files changed

Lines changed: 54 additions & 5 deletions

File tree

crates/ultradag-node/src/main.rs

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,44 +125,59 @@ async fn load_state(server: &NodeServer, data_dir: &std::path::Path) {
125125
let state_path = data_dir.join("state.json");
126126
let mempool_path = data_dir.join("mempool.json");
127127

128+
info!("Loading state from: {}", data_dir.display());
129+
128130
if BlockDag::exists(&dag_path) {
129131
match BlockDag::load(&dag_path) {
130132
Ok(dag) => {
133+
let current_round = dag.current_round();
134+
info!("Loaded DAG from disk: current_round={}", current_round);
131135
*server.dag.write().await = dag;
132-
info!("Loaded DAG from disk");
133136
}
134137
Err(e) => warn!("Failed to load DAG: {}", e),
135138
}
139+
} else {
140+
info!("No DAG state file found, starting fresh");
136141
}
137142

138143
if FinalityTracker::exists(&finality_path) {
139144
match FinalityTracker::load(&finality_path, 3) {
140145
Ok(fin) => {
146+
let last_fin = fin.last_finalized_round();
147+
info!("Loaded finality state from disk: last_finalized_round={}", last_fin);
141148
*server.finality.write().await = fin;
142-
info!("Loaded finality state from disk");
143149
}
144150
Err(e) => warn!("Failed to load finality: {}", e),
145151
}
152+
} else {
153+
info!("No finality state file found, starting fresh");
146154
}
147155

148156
if StateEngine::exists(&state_path) {
149157
match StateEngine::load(&state_path) {
150158
Ok(state) => {
159+
let supply = state.total_supply();
160+
let last_round = state.last_finalized_round();
161+
info!("Loaded state engine from disk: total_supply={}, last_finalized_round={:?}", supply, last_round);
151162
*server.state.write().await = state;
152-
info!("Loaded state engine from disk");
153163
}
154164
Err(e) => warn!("Failed to load state: {}", e),
155165
}
166+
} else {
167+
info!("No state file found, starting fresh");
156168
}
157169

158170
if Mempool::exists(&mempool_path) {
159171
match Mempool::load(&mempool_path) {
160172
Ok(mp) => {
173+
let tx_count = mp.len();
174+
info!("Loaded mempool from disk: {} transactions", tx_count);
161175
*server.mempool.write().await = mp;
162-
info!("Loaded mempool from disk");
163176
}
164177
Err(e) => warn!("Failed to load mempool: {}", e),
165178
}
179+
} else {
180+
info!("No mempool file found, starting fresh");
166181
}
167182
}
168183

scripts/extended_monitor.sh

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ echo "" | tee -a "$LOG_FILE"
1616

1717
END_TIME=$(($(date +%s) + ($DURATION_HOURS * 3600)))
1818
SAMPLE=0
19+
PREV_ROUND=0
20+
PREV_SUPPLY=0
1921

2022
while [ $(date +%s) -lt $END_TIME ]; do
2123
SAMPLE=$((SAMPLE + 1))
@@ -24,6 +26,9 @@ while [ $(date +%s) -lt $END_TIME ]; do
2426
echo "[$TIMESTAMP] Sample #$SAMPLE" | tee -a "$LOG_FILE"
2527

2628
# Check all 4 nodes
29+
ROUND_ROLLBACK=false
30+
SUPPLY_DROP=false
31+
2732
for n in 1 2 3 4; do
2833
STATUS=$(curl -s --max-time 10 "https://ultradag-node-$n.fly.dev/status" 2>/dev/null)
2934

@@ -52,12 +57,41 @@ try:
5257
issues_str = ' [' + ','.join(issues) + ']' if issues else ''
5358
5459
print(f' Node $n: round={dag:4d} fin={fin:4d} lag={lag:2d} peers={peers:2d} supply={supply:,.0f}{issues_str} {status_icon}')
60+
61+
# Output round and supply for rollback detection (always output, will filter in shell)
62+
print(f'NODE$n:ROUND:{dag}')
63+
print(f'NODE$n:SUPPLY:{int(supply)}')
5564
except Exception as e:
5665
print(f' Node $n: Parse error - {e}')
57-
" | tee -a "$LOG_FILE"
66+
" | tee -a "$LOG_FILE" | {
67+
# Capture round and supply from node 1 output only
68+
while IFS= read -r line; do
69+
if [[ $line == NODE1:ROUND:* ]]; then
70+
CURRENT_ROUND_CHECK=${line#NODE1:ROUND:}
71+
if [ $PREV_ROUND -gt 0 ] && [ $CURRENT_ROUND_CHECK -lt $PREV_ROUND ]; then
72+
ROUND_ROLLBACK=true
73+
fi
74+
PREV_ROUND=$CURRENT_ROUND_CHECK
75+
elif [[ $line == NODE1:SUPPLY:* ]]; then
76+
CURRENT_SUPPLY=${line#NODE1:SUPPLY:}
77+
if [ $PREV_SUPPLY -gt 0 ] && [ $CURRENT_SUPPLY -lt $((PREV_SUPPLY - 100000)) ]; then
78+
SUPPLY_DROP=true
79+
fi
80+
PREV_SUPPLY=$CURRENT_SUPPLY
81+
fi
82+
done
83+
}
5884
fi
5985
done
6086

87+
# Alert on rollback or supply drop
88+
if [ "$ROUND_ROLLBACK" = true ]; then
89+
echo " 🚨 ALERT: ROUND ROLLBACK DETECTED! Network may have restarted from old state." | tee -a "$LOG_FILE"
90+
fi
91+
if [ "$SUPPLY_DROP" = true ]; then
92+
echo " 🚨 ALERT: SUPPLY DROP DETECTED! Possible state rollback or corruption." | tee -a "$LOG_FILE"
93+
fi
94+
6195
# Check vertex density for recent rounds
6296
CURRENT_ROUND=$(curl -s "https://ultradag-node-1.fly.dev/status" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['dag_round'])" 2>/dev/null)
6397

0 commit comments

Comments
 (0)