-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathphase_execution.rs
More file actions
573 lines (504 loc) · 23.4 KB
/
Copy pathphase_execution.rs
File metadata and controls
573 lines (504 loc) · 23.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
//! Phase execution, transitions, timeout handling, and recovery logic.
use std::time::Instant;
use crate::error::{SyncError, SyncResult};
use crate::network::NetworkManager;
use crate::storage::StorageManager;
use key_wallet_manager::wallet_interface::WalletInterface;
use super::manager::SyncManager;
use super::phases::SyncPhase;
impl<
S: StorageManager + Send + Sync + 'static,
N: NetworkManager + Send + Sync + 'static,
W: WalletInterface,
> SyncManager<S, N, W>
{
/// Execute the current sync phase
pub(super) async fn execute_current_phase(
&mut self,
network: &mut N,
storage: &mut S,
) -> SyncResult<()> {
match &self.current_phase {
SyncPhase::DownloadingHeaders {
..
} => {
tracing::info!("📥 Starting header download phase");
// Don't call start_sync if already prepared - just send the request
if self.header_sync.is_syncing() {
// Already prepared, just send the initial request
let base_hash = self.get_base_hash_from_storage(storage).await?;
self.header_sync.request_headers(network, base_hash).await?;
} else {
// Not prepared yet, start sync normally
self.header_sync.start_sync(network, storage).await?;
}
}
SyncPhase::DownloadingMnList {
..
} => {
tracing::info!("📥 Starting masternode list download phase");
// Get the effective chain height from header sync which accounts for checkpoint base
let effective_height = self.header_sync.get_chain_height();
let sync_base_height = self.header_sync.get_sync_base_height();
// Also get the actual tip height to verify (blockchain height)
let storage_tip = storage
.get_tip_height()
.await
.map_err(|e| SyncError::Storage(format!("Failed to get storage tip: {}", e)))?;
// Debug: Check chain state
let chain_state = storage.load_chain_state().await.map_err(|e| {
SyncError::Storage(format!("Failed to load chain state: {}", e))
})?;
let chain_state_height = chain_state.as_ref().map(|s| s.get_height()).unwrap_or(0);
tracing::info!(
"Starting masternode sync: effective_height={}, sync_base={}, storage_tip={:?}, chain_state_height={}, expected_storage_index={}",
effective_height,
sync_base_height,
storage_tip,
chain_state_height,
if sync_base_height > 0 { effective_height.saturating_sub(sync_base_height) } else { effective_height }
);
// Use the minimum of effective height and what's actually in storage
let _safe_height = if let Some(tip) = storage_tip {
let storage_based_height = tip;
if storage_based_height < effective_height {
tracing::warn!(
"Chain state height {} exceeds storage height {}, using storage height",
effective_height,
storage_based_height
);
storage_based_height
} else {
effective_height
}
} else {
effective_height
};
// Start masternode sync (unified processing)
match self.masternode_sync.start_sync(network, storage).await {
Ok(_) => {
tracing::info!("🚀 Masternode sync initiated successfully, will complete when QRInfo arrives");
}
Err(e) => {
tracing::error!("❌ Failed to start masternode sync: {}", e);
return Err(e);
}
}
}
SyncPhase::DownloadingCFHeaders {
..
} => {
tracing::info!("📥 Starting filter header download phase");
// Get sync base height from header sync
let sync_base_height = self.header_sync.get_sync_base_height();
if sync_base_height > 0 {
tracing::info!(
"Setting filter sync base height to {} for checkpoint sync",
sync_base_height
);
self.filter_sync.set_sync_base_height(sync_base_height);
}
// Use flow control if enabled, otherwise use single-request mode
let sync_started = if self.config.enable_cfheaders_flow_control {
tracing::info!("Using CFHeaders flow control for parallel sync");
self.filter_sync.start_sync_headers_with_flow_control(network, storage).await?
} else {
tracing::info!("Using single-request CFHeaders sync (flow control disabled)");
self.filter_sync.start_sync_headers(network, storage).await?
};
if !sync_started {
// No peers support compact filters or already up to date
tracing::info!("Filter header sync not started (no peers support filters or already synced)");
// Transition to next phase immediately
self.transition_to_next_phase(
storage,
network,
"Filter sync skipped - no peer support",
)
.await?;
// Return early to let the main sync loop execute the next phase
return Ok(());
}
}
SyncPhase::DownloadingFilters {
..
} => {
tracing::info!("📥 Starting filter download phase");
// Get the range of filters to download
// Note: get_filter_tip_height() now returns absolute blockchain height
let filter_header_tip = storage
.get_filter_tip_height()
.await
.map_err(|e| SyncError::Storage(format!("Failed to get filter tip: {}", e)))?
.unwrap_or(0);
// No filter headers available, skip to next phase
if filter_header_tip == 0 {
self.transition_to_next_phase(storage, network, "No filter headers available")
.await?;
return Ok(());
}
tracing::info!(
"🔍 Filter download check: filter_header_tip={}, sync_base_height={}",
filter_header_tip,
self.header_sync.get_sync_base_height()
);
// Check what filters are already stored to resume download
let stored_filter_height =
storage.get_stored_filter_height().await.map_err(|e| {
SyncError::Storage(format!("Failed to get stored filter height: {}", e))
})?;
tracing::info!(
"🔍 Stored filter height from disk scan: {:?}",
stored_filter_height
);
// Resume from the next height after the last stored filter
// If no filters are stored, start from sync_base_height or 1
let start_height = if let Some(stored_height) = stored_filter_height {
tracing::info!(
"Found stored filters up to height {}, resuming from height {}",
stored_height,
stored_height + 1
);
stored_height + 1
} else {
let base_height = self.header_sync.get_sync_base_height().max(1);
tracing::info!("No stored filters found, starting from height {}", base_height);
base_height
};
// If we've already downloaded all filters, skip to next phase
if start_height > filter_header_tip {
tracing::info!(
"All filters already downloaded (stored up to {}, tip is {}), skipping to next phase",
start_height - 1,
filter_header_tip
);
self.transition_to_next_phase(storage, network, "Filters already synced")
.await?;
return Ok(());
}
let count = filter_header_tip - start_height + 1;
tracing::info!(
"Starting filter download from height {} to {} ({} filters)",
start_height,
filter_header_tip,
count
);
// Update the phase to track the expected total
if let SyncPhase::DownloadingFilters {
total_filters,
..
} = &mut self.current_phase
{
*total_filters = count;
}
// Use the filter sync manager to download filters
self.filter_sync
.sync_filters_with_flow_control(
network,
storage,
Some(start_height),
Some(count),
)
.await?;
}
SyncPhase::DownloadingBlocks {
..
} => {
tracing::info!("📥 Starting block download phase");
// Block download will be initiated based on filter matches
// For now, we'll complete the sync
self.transition_to_next_phase(storage, network, "No blocks to download").await?;
}
_ => {
// Idle or FullySynced - nothing to execute
}
}
Ok(())
}
/// Transition to the next phase
pub(super) async fn transition_to_next_phase(
&mut self,
storage: &mut S,
network: &N,
reason: &str,
) -> SyncResult<()> {
// Get the next phase
let next_phase =
self.transition_manager.get_next_phase(&self.current_phase, storage, network).await?;
if let Some(next) = next_phase {
// Check if transition is allowed
if !self
.transition_manager
.can_transition_to(&self.current_phase, &next, storage)
.await?
{
return Err(SyncError::Validation(format!(
"Invalid phase transition from {} to {}",
self.current_phase.name(),
next.name()
)));
}
// Create transition record
let transition = self.transition_manager.create_transition(
&self.current_phase,
&next,
reason.to_string(),
);
tracing::info!(
"🔄 Phase transition: {} → {} (reason: {})",
transition.from_phase,
transition.to_phase,
transition.reason
);
// Log final progress of the phase
if let Some(ref progress) = transition.final_progress {
tracing::info!(
"📊 Phase {} completed: {} items in {:?} ({:.1} items/sec)",
transition.from_phase,
progress.items_completed,
progress.elapsed,
progress.rate
);
}
self.phase_history.push(transition);
self.current_phase = next;
self.current_phase_retries = 0;
// Start the next phase
// Note: We can't execute the next phase here as we don't have network access
// The caller will need to execute the next phase
} else {
tracing::info!("✅ Sequential sync complete!");
// Calculate total sync stats
if let Some(start_time) = self.sync_start_time {
let total_time = start_time.elapsed();
let headers_synced = self.calculate_total_headers_synced();
let filters_synced = self.calculate_total_filters_synced();
let blocks_downloaded = self.calculate_total_blocks_downloaded();
self.current_phase = SyncPhase::FullySynced {
sync_completed_at: Instant::now(),
total_sync_time: total_time,
headers_synced,
filters_synced,
blocks_downloaded,
};
tracing::info!(
"🎉 Sync completed in {:?} - {} headers, {} filters, {} blocks",
total_time,
headers_synced,
filters_synced,
blocks_downloaded
);
}
}
Ok(())
}
/// Check for timeouts and handle recovery
pub async fn check_timeout(&mut self, network: &mut N, storage: &mut S) -> SyncResult<()> {
// First check if the current phase needs to be executed (e.g., after a transition)
if self.current_phase_needs_execution() {
tracing::info!("Executing phase {} after transition", self.current_phase.name());
self.execute_current_phase(network, storage).await?;
return Ok(());
}
if let Some(last_progress) = self.current_phase.last_progress_time() {
if last_progress.elapsed() > self.phase_timeout {
tracing::warn!(
"⏰ Phase {} timed out after {:?}",
self.current_phase.name(),
self.phase_timeout
);
// Attempt recovery
self.recover_from_timeout(network, storage).await?;
}
}
// Also check phase-specific timeouts
match &self.current_phase {
SyncPhase::DownloadingHeaders {
..
} => {
self.header_sync.check_sync_timeout(storage, network).await?;
}
SyncPhase::DownloadingCFHeaders {
..
} => {
if self.config.enable_cfheaders_flow_control {
self.filter_sync.check_cfheader_request_timeouts(network, storage).await?;
} else {
self.filter_sync.check_sync_timeout(storage, network).await?;
}
}
SyncPhase::DownloadingMnList {
..
} => {
self.masternode_sync.check_sync_timeout(storage, network).await?;
// After checking timeout, see if sync completed (either normally or via timeout)
if !self.masternode_sync.is_syncing() {
tracing::info!("Masternode sync completed (detected in timeout check), transitioning to next phase");
self.transition_to_next_phase(storage, network, "Masternode sync complete")
.await?;
self.execute_current_phase(network, storage).await?;
}
}
SyncPhase::DownloadingFilters {
..
} => {
// Always check for timed out filter requests, not just during phase timeout
self.filter_sync.check_filter_request_timeouts(network, storage).await?;
// For filter downloads, we need custom timeout handling
// since the filter sync manager's timeout is for filter headers
if let Some(last_progress) = self.current_phase.last_progress_time() {
if last_progress.elapsed() > self.phase_timeout {
tracing::warn!(
"⏰ Filter download phase timed out after {:?}",
self.phase_timeout
);
// Check if we have any active requests
let active_count = self.filter_sync.active_request_count();
let pending_count = self.filter_sync.pending_download_count();
tracing::warn!(
"Filter sync status: {} active requests, {} pending",
active_count,
pending_count
);
// First check for timed out filter requests
self.filter_sync.check_filter_request_timeouts(network, storage).await?;
// Try to recover by sending more requests if we have pending ones
if self.filter_sync.has_pending_filter_requests() && active_count < 10 {
tracing::info!("Attempting to recover by sending more filter requests");
self.filter_sync.send_next_filter_batch(network).await?;
self.current_phase.update_progress();
} else if active_count == 0
&& !self.filter_sync.has_pending_filter_requests()
{
// No active requests and no pending - we're stuck
tracing::error!(
"Filter sync stalled with no active or pending requests"
);
// Check if we received some filters but not all
let received_count = self.filter_sync.get_received_filter_count();
if let SyncPhase::DownloadingFilters {
total_filters,
..
} = &self.current_phase
{
if received_count > 0 && received_count < *total_filters {
tracing::warn!(
"Filter sync stalled at {}/{} filters - attempting recovery",
received_count, total_filters
);
// Retry the entire filter sync phase
self.current_phase_retries += 1;
if self.current_phase_retries <= self.max_phase_retries {
tracing::info!(
"🔄 Retrying filter sync (attempt {}/{})",
self.current_phase_retries,
self.max_phase_retries
);
// Clear the filter sync state and restart
self.filter_sync.reset();
self.filter_sync.set_syncing_filters(false); // Allow restart
// Update progress to prevent immediate timeout
self.current_phase.update_progress();
// Re-execute the phase
self.execute_current_phase(network, storage).await?;
return Ok(());
} else {
tracing::error!(
"Filter sync failed after {} retries, forcing completion",
self.max_phase_retries
);
}
}
}
// Force transition to next phase to avoid permanent stall
self.transition_to_next_phase(
storage,
network,
"Filter sync timeout - forcing completion",
)
.await?;
self.execute_current_phase(network, storage).await?;
}
}
}
}
_ => {}
}
Ok(())
}
/// Recover from a timeout
async fn recover_from_timeout(&mut self, network: &mut N, storage: &mut S) -> SyncResult<()> {
self.current_phase_retries += 1;
if self.current_phase_retries > self.max_phase_retries {
return Err(SyncError::Timeout(format!(
"Phase {} failed after {} retries",
self.current_phase.name(),
self.max_phase_retries
)));
}
tracing::warn!(
"🔄 Retrying phase {} (attempt {}/{})",
self.current_phase.name(),
self.current_phase_retries,
self.max_phase_retries
);
// Update progress time to prevent immediate re-timeout
self.current_phase.update_progress();
// Execute phase-specific recovery
match &self.current_phase {
SyncPhase::DownloadingHeaders {
..
} => {
self.header_sync.check_sync_timeout(storage, network).await?;
}
SyncPhase::DownloadingMnList {
..
} => {
self.masternode_sync.check_sync_timeout(storage, network).await?;
}
SyncPhase::DownloadingCFHeaders {
..
} => {
if self.config.enable_cfheaders_flow_control {
self.filter_sync.check_cfheader_request_timeouts(network, storage).await?;
} else {
self.filter_sync.check_sync_timeout(storage, network).await?;
}
}
_ => {
// For other phases, we'll need phase-specific recovery
}
}
Ok(())
}
// Helper methods for calculating totals
pub(super) fn calculate_total_headers_synced(&self) -> u32 {
self.phase_history
.iter()
.find(|t| t.from_phase == "Downloading Headers")
.and_then(|t| t.final_progress.as_ref())
.map(|p| p.items_completed)
.unwrap_or(0)
}
pub(super) fn calculate_total_filters_synced(&self) -> u32 {
self.phase_history
.iter()
.find(|t| t.from_phase == "Downloading Filters")
.and_then(|t| t.final_progress.as_ref())
.map(|p| p.items_completed)
.unwrap_or(0)
}
pub(super) fn calculate_total_blocks_downloaded(&self) -> u32 {
self.phase_history
.iter()
.find(|t| t.from_phase == "Downloading Blocks")
.and_then(|t| t.final_progress.as_ref())
.map(|p| p.items_completed)
.unwrap_or(0)
}
pub(super) fn no_more_pending_blocks(&self) -> bool {
// This would check if there are more blocks to download
// For now, return true
true
}
}