-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathserver.js
More file actions
3617 lines (3408 loc) · 174 KB
/
Copy pathserver.js
File metadata and controls
3617 lines (3408 loc) · 174 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Sentinel dVPN Network Audit — Server
* Thin Express server: API routes, SSE, imports from modular architecture.
*/
import express from 'express';
import cookieParser from 'cookie-parser';
import path from 'path';
import { fileURLToPath } from 'url';
import { EventEmitter } from 'events';
import { existsSync } from 'fs';
import { adminOnly, attachAdminFlag, safeEq, setAdminSessionValidator } from './core/auth.js';
import { rateLimit, sseLimit } from './core/rate-limit.js';
import { MNEMONIC, DENOM, GAS_PRICE, PORT, LCD_ENDPOINTS, RPC_ENDPOINTS, PROJECT_ROOT, DNS_PRESETS, ACTIVE_DNS, setActiveDns } from './core/constants.js';
import { getSettings, updateSettings, getDefaultSettings } from './core/settings.js';
import { queryReports as queryOnchainReports } from './core/onchain-report.js';
import { cachedWalletSetup, createFreshClient } from './core/wallet.js';
import { ensureLcd, getActiveLcd, cleanupRpc, getAllNodes } from './core/chain.js';
import { nodeStatusV3 } from './protocol/v3protocol.js';
import { createState, runAudit, runRetestSkips, runPlanTest, runSubPlanTest, getResults, saveResults, triggerPipelineStop } from './audit/pipeline.js';
import {
insertRun, updateRunOnFinish, getRunSpendByFinish, getRun,
insertResult, insertErrorLog,
searchNodes, getNodeDetail, getNodeErrors, getCountryList,
getActiveRun, getLastCompletedRun, getBandwidthHistory,
searchErrors, getNetworkStats, getRunStats,
listBatches, getBatchResults, getActiveBatch, getLastBatch,
insertBatch, updateBatchOnFinish, insertBatchResult,
reopenBatch, getBatchById, getBatchWithNodes,
getDb,
} from './core/db.js';
import * as continuous from './audit/continuous.js';
import { getInstalledVersions, verifyAllSdks, verifySdk } from './core/sdk-verify.js';
// Force line-buffered stdout/stderr so boot diagnostics flush immediately
// even when redirected to a file (Windows defaults to block-buffering, which
// hides every console.log if the process hangs before app.listen).
try { process.stdout._handle?.setBlocking?.(true); } catch (e) { console.error('[boot] stdout setBlocking failed:', e.message); }
try { process.stderr._handle?.setBlocking?.(true); } catch (e) { console.error('[boot] stderr setBlocking failed:', e.message); }
// Platform-aware WireGuard import — Windows / Linux / macOS each have full implementations
// Wrapped in a 5s timeout: the windows module runs sync `execSync` probes
// (`net session`, `where wireguard.exe`, `wg-quick --version`) at its own
// module scope. Any of those can stall on a slow Service Control Manager
// and deadlock the entire server boot. Falling back to WG_AVAILABLE=false
// is preferable to a silent zombie.
let emergencyCleanupSync, watchdogCheck, WG_AVAILABLE, IS_ADMIN;
const _wgFallback = () => {
console.error('[boot] WireGuard module init timed out — continuing with WG disabled');
emergencyCleanupSync = () => {};
watchdogCheck = () => {};
WG_AVAILABLE = false;
IS_ADMIN = false;
};
const _wgImport = (() => {
if (process.platform === 'win32') return import('./platforms/windows/wireguard.js');
if (process.platform === 'linux') return import('./platforms/linux/wireguard.js');
if (process.platform === 'darwin') return import('./platforms/macos/wireguard.js');
return null;
})();
if (_wgImport) {
const _wgTimeout = new Promise((_, reject) => setTimeout(() => reject(new Error('wg-import-timeout')), 5000));
try {
({ emergencyCleanupSync, watchdogCheck, WG_AVAILABLE, IS_ADMIN } = await Promise.race([_wgImport, _wgTimeout]));
} catch (e) {
console.error(`[boot] WireGuard import failed: ${e.message}`);
_wgFallback();
}
} else {
emergencyCleanupSync = () => {};
watchdogCheck = () => {};
WG_AVAILABLE = false;
IS_ADMIN = process.getuid?.() === 0 || false;
}
import { loadTransportCache, getCacheStats } from './core/transport-cache.js';
import { DirectSecp256k1HdWallet } from '@cosmjs/proto-signing';
import { SigningStargateClient, GasPrice } from '@cosmjs/stargate';
// Walk RPC_ENDPOINTS in order and return the first SigningStargateClient that
// connects. Replaces a hardcoded `rpc.sentinel.co:443` connect that returned
// stale balances when that node fell behind tip while reporting catching_up=false.
async function connectWithRpcFailover(wallet) {
const opts = { gasPrice: GasPrice.fromString(GAS_PRICE) };
let lastErr;
for (const url of RPC_ENDPOINTS) {
try {
return await SigningStargateClient.connectWithSigner(url, wallet, opts);
} catch (e) {
lastErr = e;
}
}
throw lastErr || new Error('All RPC endpoints unreachable');
}
const __dirname = path.dirname(fileURLToPath(import.meta.url));
process.env.PATH = path.join(__dirname, 'bin') + path.delimiter + (process.env.PATH || '');
// ─── Env sanity check ───────────────────────────────────────────────────────
const PUBLIC_MODE = process.env.PUBLIC_MODE === 'true';
const ADMIN_PATH = process.env.ADMIN_PATH || '/admin';
const ADMIN_TOKEN = process.env.ADMIN_TOKEN || '';
// M-05: use ephemeral per-process secret when ADMIN_TOKEN is absent; forbids
// forged signed cookies even in single-user/dev mode.
import crypto from 'node:crypto';
const COOKIE_SECRET = ADMIN_TOKEN || crypto.randomBytes(32).toString('hex');
// ─── Admin session store (H-02) ─────────────────────────────────────────────
// Map<sessionId, expiryMs>. Session ID is stored in the signed cookie instead
// of the raw ADMIN_TOKEN so cookie theft cannot recover the backend token.
// In-memory only: admin logouts drop entries; process restart invalidates all sessions.
const ADMIN_SESSIONS = new Map();
const ADMIN_SESSION_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
// Cap to bound memory under sustained brute-force or buggy clients that never
// log out. Map iteration order is insertion order — drop the oldest entry when
// we exceed the cap. 1000 sessions × ~80 bytes = ~80 KB worst case.
const ADMIN_SESSIONS_MAX = 1000;
export function createAdminSession() {
const id = crypto.randomBytes(32).toString('hex');
if (ADMIN_SESSIONS.size >= ADMIN_SESSIONS_MAX) {
const oldest = ADMIN_SESSIONS.keys().next().value;
if (oldest) ADMIN_SESSIONS.delete(oldest);
}
ADMIN_SESSIONS.set(id, Date.now() + ADMIN_SESSION_TTL_MS);
return id;
}
export function isValidAdminSession(id) {
if (!id || typeof id !== 'string') return false;
const exp = ADMIN_SESSIONS.get(id);
if (!exp) return false;
if (exp < Date.now()) { ADMIN_SESSIONS.delete(id); return false; }
return true;
}
export function revokeAdminSession(id) {
if (id) ADMIN_SESSIONS.delete(id);
}
// Periodic cleanup of expired sessions — 1-hour interval
setInterval(() => {
const now = Date.now();
for (const [id, exp] of ADMIN_SESSIONS) {
if (exp < now) ADMIN_SESSIONS.delete(id);
}
}, 60 * 60 * 1000).unref();
// Inject the validator into the auth middleware. Must run before any admin request.
setAdminSessionValidator(isValidAdminSession);
if (PUBLIC_MODE && !ADMIN_TOKEN) {
console.error('');
console.error('ERROR: PUBLIC_MODE=true requires ADMIN_TOKEN to be set.');
console.error(' Without ADMIN_TOKEN, the admin surface has no protection.');
console.error(' Generate one: node -e "console.log(require(\'crypto\').randomBytes(32).toString(\'hex\'))"');
console.error(' Then add ADMIN_TOKEN=<value> to your .env file.');
console.error('');
process.exit(1);
}
if (!MNEMONIC || !MNEMONIC.trim()) {
console.warn('');
console.warn('⚠ MNEMONIC is not set.');
console.warn(' The server will start, but any test that signs a TX will fail.');
console.warn(' Fix: copy .env.example to .env and set MNEMONIC to a 12-word Cosmos phrase.');
console.warn('');
}
// ─── WireGuard Safety: cleanup on ANY exit ──────────────────────────────────
// Boot-time cleanup is deferred until AFTER app.listen — running it inline
// here can block the event loop for 10–30s on a slow Service Control Manager
// (sc query / sc stop / sc delete each carry their own 5s timeouts).
function onProcessExit() { cleanupRpc(); emergencyCleanupSync(); }
process.on('exit', onProcessExit);
// Graceful shutdown: stop the continuous loop before exit so it can't keep
// writing `runs` rows after the HTTP listener closes. Best-effort only; the
// hard exit fires after 2s regardless so Ctrl-C is still snappy.
function gracefulShutdown(signal, exitCode) {
console.log(`[server] ${signal} received — stopping continuous loop`);
// Force a final snapshot before teardown so a Ctrl-C / SIGTERM within the
// throttle window still persists the latest in-flight state for resume.
try { flushStateSnapshot(); } catch (e) { console.error('[shutdown] snapshot flush failed:', e.message); }
try { continuous.stop(); } catch {}
onProcessExit();
setTimeout(() => process.exit(exitCode), 2_000).unref();
}
process.on('SIGINT', () => gracefulShutdown('SIGINT', 130));
process.on('SIGTERM', () => gracefulShutdown('SIGTERM', 143));
// Crash loud, crash fast. Without process.exit, the handler runs cleanup and
// the event loop keeps going on a half-initialised state — silent zombie.
process.on('uncaughtException', (err) => {
const msg = err?.stack || err?.message || String(err);
console.error(`[uncaughtException] ${msg}`);
try { emergencyCleanupSync(); } catch (e) { console.error('[uncaughtException] cleanup failed:', e.message); }
setTimeout(() => process.exit(1), 500).unref();
});
process.on('unhandledRejection', (reason) => {
const msg = reason?.stack || reason?.message || String(reason);
console.error(`[unhandledRejection] ${msg}`);
try { emergencyCleanupSync(); } catch (e) { console.error('[unhandledRejection] cleanup failed:', e.message); }
setTimeout(() => process.exit(1), 500).unref();
});
// Watchdog: every 5s, check if a tunnel has been up too long
setInterval(() => {
if (watchdogCheck()) {
broadcast('log', { msg: '⚠ WATCHDOG: Force-killed stale WireGuard tunnel' });
}
}, 5_000);
// ─── SSE ────────────────────────────────────────────────────────────────────
const emitter = new EventEmitter();
emitter.setMaxListeners(100);
// Sized to comfortably hold the full log of a typical run (~10–20 lines per
// node × hundreds of nodes) plus headroom. SSE init replays this to admin and
// live so a refresh / reconnect / resume sees the run's full prior history,
// not just the last few lines.
const LOG_BUFFER_MAX = 5000;
// The full LOG_BUFFER_MAX backlog is kept server-side, but the SSE `init`
// bootstrap frame replays only the most recent INIT_LOG_REPLAY lines. A fresh
// tab needs recent context, not 5000 lines — replaying the whole buffer made
// the init frame hundreds of KB. /live is unaffected: it separately refetches
// the full backlog from GET /api/public/logs (live.html), which still returns
// the whole (filtered) buffer. admin.html relies on the init replay, so the
// admin log pane shows the most recent INIT_LOG_REPLAY lines on a fresh
// connect/refresh; subsequent live lines stream in normally as events arrive.
const INIT_LOG_REPLAY = 500;
const logBuffer = [];
// ─── State Snapshot (persists volatile fields across restarts) ───────────────
const STATE_SNAPSHOT_FILE = path.join(__dirname, 'results', '.state-snapshot.json');
let _lastSnapshotTs = 0;
function saveStateSnapshot(force = false) {
// Throttle: save at most every 5 seconds to avoid disk thrashing — unless
// `force` is set. Terminal status transitions (stop/done/error) and explicit
// flush points pass force=true so a crash/stop within the 5s window can't
// lose the latest activeBatchId / spend / resumeHeadAddr / activeDbRunId.
const now = Date.now();
if (!force && now - _lastSnapshotTs < 5_000) return;
_lastSnapshotTs = now;
try {
_wfs(STATE_SNAPSHOT_FILE, JSON.stringify({
baselineHistory: state.baselineHistory,
nodeSpeedHistory: state.nodeSpeedHistory,
spentUdvpn: state.spentUdvpn,
runSpentUdvpn: state.runSpentUdvpn ?? 0,
balanceUdvpn: state.balanceUdvpn,
balance: state.balance,
estimatedTotalCost: state.estimatedTotalCost,
startedAt: state.startedAt,
baselineMbps: state.baselineMbps,
totalNodes: state.totalNodes,
status: state.status,
// Run-mode context — without this, /api/resume after a process bounce
// silently demotes a subscription run to P2P (the C-1 family of bugs).
runMode: state.runMode,
testRun: state.testRun,
runPlanId: state.runPlanId,
runSubscriptionId: state.runSubscriptionId,
runGranter: state.runGranter,
pricingMode: state.pricingMode,
activeSDK: state.activeSDK,
continuousLoop: state.continuousLoop,
// Persist the open batch handle so /api/resume after a process bounce
// can re-attach to the same batches row instead of starting a new one.
activeBatchId: state.activeBatchId || 0,
// Address of the in-flight node when stop hit, so resume can hoist
// it back to the front of the next scan order.
resumeHeadAddr: state.resumeHeadAddr || null,
// Path of the audit log file currently being appended to. Survives
// process bounce so /api/resume reuses the same file instead of
// creating a fresh `audit-<ts>.log` and orphaning prior entries.
auditLogPath: state.auditLogPath || null,
// SQLite runs.id of the in-flight run. Without this, /api/resume after
// a process bounce leaves state.activeDbRunId=null and post-resume failures
// skip insertErrorLog — the node-detail popup then has nothing to show.
activeDbRunId: state.activeDbRunId || null,
// The run currently displayed (dropdown selection + save/resume dir). Must
// survive a bounce so boot doesn't re-alias an unsaved run onto a saved one.
activeRunNumber: state.activeRunNumber ?? null,
// H-1: read-only marker for a loaded historical run. Without persisting +
// restoring this, a process bounce clears it and /api/resume would let an
// incomplete loaded run be resumed (appending live rows onto a past run).
loadedReadonly: !!state.loadedReadonly,
}), 'utf8');
} catch (e) { console.error('[snapshot] write failed:', e.message); }
}
// Force a non-throttled snapshot write. Call on terminal status transitions
// and in /api/stop so the latest volatile fields survive a stop/crash that
// lands inside the 5s throttle window.
function flushStateSnapshot() { saveStateSnapshot(true); }
// ─── Log categorization ──────────────────────────────────────────────────────
// Every log line gets exactly ONE category: 'events' | 'sys' | 'node'.
// EVENTS — operator/lifecycle (start/stop/save/load, on-chain, wallet, DNS…)
// SYS — in-run diagnostics (baseline, balance, connectivity, scan…)
// NODE — per-node results incl. failures (default).
// First match wins, checked in EVENTS → SYS → NODE order. The 📡 emoji is shared
// by on-chain (events) and baseline (sys), so classify by the WORDS, not emoji.
// IMPORTANT: keep these keyword lists byte-identical to admin.html's logCategory().
function classifyLogCategory(msg) {
const s = String(msg == null ? '' : msg);
// NOTE: no bare '💾' — it also prefixes the per-node "💾 Cached:" transport
// line; 'Saved' already covers the lifecycle save lines. 'Resuming Test' is a
// lifecycle sibling of 'Starting Test'.
const EVENTS = ['Starting Test', 'Resuming Test', 'Stop requested', '⏹', 'Loop continues', '♾', 'Deleted Test', '🗑', 'Saved', 'Loaded Test', '📂', 'DNS', '🔧', 'On-chain reporting', 'On-chain report posted', 'Setting up wallet', '🔑', 'Log file', '📝', 'subscribing', '📋', 'SDK switched', 'Broadcast'];
const SYS = ['baseline', 'Baseline', 'Balance', '💰', 'internet', 'Internet', 'connectivity', '🌐', 'Transport cache', '🧠', 'Fetching node list', '🔍', 'V2Ray:', 'WireGuard:', 'Admin:', 'Cloudflare', 'Discovered', 'active plans', 'online scan', 'Scanning'];
for (const k of EVENTS) if (s.includes(k)) return 'events';
for (const k of SYS) if (s.includes(k)) return 'sys';
return 'node';
}
// Public /live shows per-NODE activity only — operator EVENTS and in-run SYS
// diagnostics are hidden from spectators (the admin dashboard still shows all).
// Filters the rolling string buffer for the public log endpoints.
function publicLogBuffer() {
return logBuffer.filter(e => classifyLogCategory(e.msg) === 'node');
}
// Is there an ACTIVE run worth showing to /live right now? Public live surfaces
// (SSE init, /logs, /live-state) must return EMPTY work payloads when no run is
// in flight — otherwise a paused/idle page (e.g. right after boot, when
// logBuffer is hydrated from results/audit-*.log) ships the last run's log
// backlog + results behind the "Testing Has Been Paused" overlay. The overlay
// is opaque so it's not visible, but the page still LOADS that data — which is
// exactly what we don't want. Gate the payload at the source instead.
function publicRunActive() {
try { if (continuous.status()?.running) return true; } catch (e) { console.error('[publicRunActive] continuous.status failed:', e.message); }
try { if (getActiveBatch()) return true; } catch (e) { console.error('[publicRunActive] getActiveBatch failed:', e.message); }
const st = state && state.status;
return st === 'running' || (typeof st === 'string' && st.indexOf('paused') === 0);
}
// EVENTS persist to a file (separate from per-run runs/test-NNN/audit.log), with
// a simple 1-file rotation at ~2MB so it can't grow unbounded.
const EVENTS_LOG_FILE = path.join(__dirname, 'results', 'events.log');
const EVENTS_LOG_MAX_BYTES = 2 * 1024 * 1024;
function appendEventLog(msg) {
try {
try {
const st = _statSync(EVENTS_LOG_FILE);
if (st && st.size > EVENTS_LOG_MAX_BYTES) {
_rfs2(EVENTS_LOG_FILE, EVENTS_LOG_FILE + '.1');
}
} catch (e) {
// ENOENT (no file yet) is expected — only log genuine stat/rotate errors.
if (e && e.code !== 'ENOENT') console.error('[events.log] rotate failed:', e.message);
}
_afs(EVENTS_LOG_FILE, `[${new Date().toISOString()}] ${msg}\n`, 'utf8');
} catch (e) {
console.error('[events.log] append failed:', e.message);
}
}
// ─── Server-authoritative ETA (windowed real-throughput) ───────────────────
// Both admin.html and live.html used to compute ETA independently from
// different inputs, so they disagreed. The server now owns the single source
// of truth: it keeps a rolling window of the last ETA_WINDOW node-completion
// timestamps, derives the current completion RATE (nodes/ms), and broadcasts
// a REMAINING DURATION (`etaRemainingMs`) — NOT an absolute epoch. Each client
// anchors that duration to its OWN clock at receipt and counts it down, so a
// skewed browser clock can't distort the ETA (an absolute server epoch rendered
// against a skewed client Date.now() drifted by the offset — minutes). The math
// is throughput-based (parallelism-aware), not per-node-duration based.
const ETA_WINDOW = 12;
let _etaCompletions = []; // recent node-completion epoch ms (rolling, current pass)
let _etaLastDone = -1; // last seen completed-count, to detect a new pass
let _etaPrevStatus = null; // last seen status, to detect a fresh →running transition
// Resolve the (done, total) pair the ETA should measure against. During a
// retest (runRetestSkips), every node already has a result row so the whole-run
// counters are saturated (done ≈ total) and remaining ≤ 0 — the ETA would pin
// at 00:00:00 for the entire retest. The retest's REAL progress lives in
// separate fields (retestTested / retestPassed+retestFailed / retestTotal), so
// when retestMode is set with a usable retestTotal we measure against those
// instead. Falls back to the whole-run counters whenever the retest fields are
// absent or non-positive (defensive — never let a missing field zero the ETA).
function etaProgress(st) {
if (st.retestMode && Number(st.retestTotal) > 0) {
// retestTested already counts every retested node (pass + fail), so it is
// the authoritative "done" for the retest pass; fall back to summing
// retestPassed+retestFailed if retestTested is somehow absent.
let done = Number(st.retestTested);
if (!(done >= 0)) done = (Number(st.retestPassed) || 0) + (Number(st.retestFailed) || 0);
return { done, total: Number(st.retestTotal) };
}
const done = (st.testedNodes || 0) + (st.failedNodes || 0) + (st.skippedNodes || 0);
const total = st.totalNodes || 0;
return { done, total };
}
// Returns the REMAINING milliseconds until the active pass completes, or null
// when not computable (not running / <2 completions / total<=0). 0 when there
// is no work left. Clients anchor this to their own clock and tick it down.
function computeEtaRemainingMs(st) {
if (!st || typeof st !== 'object') return null;
// Only meaningful while a pass is actively running. done/idle/paused_* → null.
if (st.status !== 'running') return null;
const { done, total } = etaProgress(st);
if (total <= 0) return null;
const remaining = total - done;
if (remaining <= 0) return 0; // no work left → renders 00:00:00
if (_etaCompletions.length < 2) return null; // not enough data → client shows "Calculating…"
const span = _etaCompletions[_etaCompletions.length - 1] - _etaCompletions[0];
if (span <= 0) return null;
const n = _etaCompletions.length - 1; // intervals across the window
const ratePerMs = n / span;
const etaMs = remaining / ratePerMs;
return Math.round(etaMs);
}
function broadcast(type, data = {}) {
if (type === 'log' && data.msg) {
// Tag the live SSE 'log' event with a category so admin/live can filter
// without re-deriving it. Set BEFORE emitter.emit below.
data.cat = data.cat || classifyLogCategory(data.msg);
// Stamp emission time ONCE, here, so every surface shows WHEN a line was
// produced — not when the client rendered/replayed it. The ts rides on the
// live SSE event (data.ts), the in-memory buffer ({ts,msg}), and — via the
// pipeline's ISO-prefixed logLine — the on-disk audit log, so it survives a
// restart too. Buffer entries are {ts,msg} objects; consumers read .msg.
if (data.ts == null) data.ts = Date.now();
logBuffer.push({ ts: data.ts, msg: data.msg });
if (logBuffer.length > LOG_BUFFER_MAX) logBuffer.shift();
if (data.cat === 'events') appendEventLog(data.msg);
}
if (type === 'state' || type === 'result') saveStateSnapshot();
// ─── ETA bookkeeping ──────────────────────────────────────────────────────
// Record the just-finished node FIRST (so it's in the window), then compute
// the remaining duration and stamp it onto any state payload so admin (which
// ticks on 'result' events) and live (which ticks on 'state') both anchor the
// same value to their own clocks.
if (type === 'result') {
_etaCompletions.push(Date.now());
if (_etaCompletions.length > ETA_WINDOW) {
_etaCompletions = _etaCompletions.slice(-ETA_WINDOW);
}
}
if (data && data.state && typeof data.state === 'object') {
const s = data.state;
// Explicit fresh-run ring reset: when status transitions INTO 'running' from
// any non-running status, wipe the window so a new run can never inherit the
// prior run's completion timestamps. This no longer relies on an incidental
// zero-`done` broadcast landing before the first result of the new run.
if (s.status === 'running' && _etaPrevStatus !== 'running') {
_etaCompletions = [];
_etaLastDone = -1;
}
_etaPrevStatus = s.status;
const { done } = etaProgress(s);
// Belt-and-suspenders per-pass reset within a continuous loop: each
// iteration's counters reset to 0, so a drop below the last-seen done count
// means a new pass started — clear the window.
if (done < _etaLastDone) _etaCompletions = [];
_etaLastDone = done;
// NOTE: for most callers data.state IS the long-lived global `state` object,
// so this assignment MUTATES the global in place — etaRemainingMs is not a
// per-payload-only field. That's fine: it's recomputed on every broadcast,
// and computeEtaRemainingMs SELF-CLEARS it to null whenever status !==
// 'running' (done/idle/paused_*), so a stale value can never linger on the
// global. saveStateSnapshot's allowlist also excludes it, so it never persists.
data.state.etaRemainingMs = computeEtaRemainingMs(data.state);
}
// NOTE: spread `data` FIRST so a payload field named `type` (e.g. the node's
// service-type like 'wireguard') cannot clobber the SSE event type. The
// event type is the dispatch key — clients switch on d.type — so it must win.
emitter.emit('update', { ...data, type });
}
// Use this for any state change where the client must replace its row table:
// run start, /api/clear, retest, load. The admin client treats `msg.results`
// presence as the wipe signal — omitting it leaves stale rows on screen, which
// has burned us before (5 stale TEST_RUN_SKIP rows after New Test).
function broadcastStateFresh(extra = {}) {
broadcast('state', { state, results: getResults(), ...extra });
}
// ─── Batch persistence wrapper for direct pipeline calls ────────────────────
// audit/continuous.js writes batches/batch_results for continuous-loop runs.
// Direct pipeline calls (subscription start/resume, p2p start/resume, plan-test,
// retest) bypass continuous.js entirely — without this wrapper they never
// produce a `batches` row, so /api/public/runs/current returns 404 and the
// /live page can't reconstruct in-flight progress on refresh.
//
// `mode` MUST match the run intent so the dashboard never confuses subscription
// (Plan #N), p2p (pay-per-GB), and test (TEST_RUN_SKIP) runs.
function withBatchTracking(baseBroadcast, mode, opts = {}) {
// Resume re-attaches to the previously-open batch via opts.existingBatchId so
// /live's hydrate-from-DB path returns the full pre-pause + post-resume row
// set as one batch. Without this, every resume opens a fresh batches row and
// the table on /live wipes back to whatever was tested AFTER resume only.
let batchId = opts.existingBatchId ? Number(opts.existingBatchId) : 0;
let opened = batchId > 0;
let closed = false;
let startEmitted = false;
if (opened) {
state.activeBatchId = batchId;
try { reopenBatch(batchId, 'real'); } catch (e) { console.error('[withBatchTracking reopen]', e.message); }
}
return function tracked(type, data = {}) {
try {
if (!closed && !opened && type === 'result' && data && data.result) {
batchId = insertBatch({
started_at: Date.now(),
snapshot_size: state.totalNodes || 0,
mode,
}, 'real');
state.activeBatchId = batchId;
opened = true;
}
if (opened && !closed && type === 'result' && data && data.result) {
const r = data.result;
insertBatchResult(batchId, {
address: r.address || '',
moniker: r.moniker || null,
country: r.country || null,
country_code: r.countryCode || r.country_code || null,
city: r.city || null,
type: r.type || null,
actual_mbps: r.actualMbps ?? null,
peers: r.peers ?? null,
max_peers: r.maxPeers ?? null,
error: r.error ? String(r.error).slice(0, 200) : null,
error_code: r.errorCode || null,
tested_at: Date.now(),
baseline_mbps: r.baselineAtTest ?? r.baselineMbps ?? null,
}, 'real');
// Emit batch:start once + batch:node:result per row so /live's Current
// Batch panel ticks for direct-pipeline runs (sub-plan, p2p, retest)
// exactly like continuous.js does. Without this, /live falls back on
// resultsArr.length which can desync if any 'result' SSE event is
// dropped (broadcastLive flip race, reconnect gap), leaving the
// counter stuck at the count from the last full REST hydrate.
if (!startEmitted) {
baseBroadcast('batch:start', {
batchId,
iteration: null,
startedAt: new Date().toISOString(),
snapshotSize: state.totalNodes || 0,
mode,
});
startEmitted = true;
}
baseBroadcast('batch:node:result', {
batchId,
address: r.address || '',
moniker: r.moniker || null,
country: r.country || null,
countryCode: r.countryCode || r.country_code || null,
city: r.city || null,
serviceType: r.type || null,
actualMbps: r.actualMbps ?? null,
baselineMbps: r.baselineAtTest ?? r.baselineMbps ?? null,
peers: r.peers ?? null,
maxPeers: r.maxPeers ?? null,
error: r.error ? String(r.error).slice(0, 200) : null,
errorCode: r.errorCode || null,
testedAt: Date.now(),
});
}
if (opened && !closed && type === 'state' && data && data.state) {
const status = data.state.status;
if (status === 'done' || status === 'error' || status === 'stopped') {
const passed = data.state.testedNodes || 0;
const failed = data.state.failedNodes || 0;
updateBatchOnFinish(batchId, {
finished_at: Date.now(),
passed,
failed,
}, 'real');
closed = true;
if (startEmitted) {
baseBroadcast('batch:end', {
batchId,
passed,
failed,
durationMs: null,
});
}
// Keep state.activeBatchId so /api/resume can find this batch and
// reopen it. /api/start clears it explicitly when a new test begins.
}
}
} catch (err) {
console.error(`[withBatchTracking ${mode}] ${err.message}`);
}
baseBroadcast(type, data);
};
}
// ─── Continuous Loop SSE forwarding ─────────────────────────────────────────
// Forward loop and batch events from the continuous runner into the broadcast bus.
{
const LOOP_EVENTS = [
'loop:started', 'loop:stopping', 'loop:stopped', 'loop:error',
'iteration:start', 'iteration:end',
];
for (const evt of LOOP_EVENTS) {
continuous.on(evt, (data) => broadcast(evt, data || {}));
}
const BATCH_EVENTS = ['batch:start', 'batch:node:result', 'batch:end', 'batch:gap'];
for (const evt of BATCH_EVENTS) {
continuous.on(evt, (data) => broadcast(evt, data || {}));
}
// Forward per-node log/state/result/progress from inside the continuous
// pipeline so the live dashboard mirrors the admin dashboard 1:1 during
// continuous-loop runs (not only direct /api/start runs).
const LIVE_EVENTS = ['log', 'state', 'result', 'progress'];
for (const evt of LIVE_EVENTS) {
continuous.on(evt, (data) => broadcast(evt, data || {}));
}
}
// ─── State ──────────────────────────────────────────────────────────────────
const state = createState();
// Initialize the read-only marker so the key is always present in admin SSE /
// /api/stats payloads (the admin dashboard reads state.loadedReadonly to hide
// the Resume button for loaded runs). Set true by /api/runs/load, cleared by
// startFreshRun / resume, and persisted+restored across bounces (H-1).
state.loadedReadonly = false;
// Derived flag: is the currently-loaded run already saved in the runs index?
// Drives the admin SAVE button (shown only for an UNSAVED loaded run). Maintained
// at the run-lifecycle points (startFreshRun=false, saveCurrentRun/persistActiveRun
// =true, loadRunIntoState=computed, clearActiveRunView=false, boot=computed) — same
// discipline as loadedReadonly — so it flows to the client via SSE automatically.
state.activeRunSaved = false;
// ─── Canonical "audit busy" predicates — ONE source of truth ─────────────────
// These replace ~10 hand-copied `state.status === 'running' || ...` checks that
// drifted out of sync: new pipeline pause states (paused_balance/paused_internet)
// were added but only some guards were updated, so a run parked on insufficient
// funds looked "idle" to half the guards. Always reach for these helpers.
//
// PIPELINE_BUSY_STATUSES = states where the audit pipeline holds the active run
// dir + SQLite run id and a live writer is either running or parked in a poll
// loop that RESUMES per-node writes on recovery. Launching a new run, retesting,
// deleting the run, or swapping SDK against any of these is unsafe.
// running — actively testing
// paused_balance — parked in the insufficient-funds poll loop (will resume)
// paused_internet — parked in the no-connectivity poll loop (will resume)
// paused — diagnostics-subsystem pause (protocol/diagnostics.js)
// NOTE: the periodic balance refresher's `running || paused_balance` check is a
// DIFFERENT concept ("is the pipeline doing its own balance refresh right now")
// and intentionally does NOT use this set — don't fold it in.
const PIPELINE_BUSY_STATUSES = ['running', 'paused', 'paused_balance', 'paused_internet'];
function isPipelineBusy() { return PIPELINE_BUSY_STATUSES.includes(state.status); }
// Pipeline busy OR the continuous-loop runner is active. Use where there is no
// separate continuous-takeover handling (delete, SDK swap).
function isAuditBusy() { return isPipelineBusy() || continuous.status().running; }
// Synchronous "an audit is being launched" guard. /api/start and /api/resume have
// an AWAIT (the continuous-takeover pause-poll) between the isPipelineBusy() check
// and the moment runAudit sets status='running'. Without a flag set before that
// await, two near-simultaneous starts both pass the check, mint duplicate run
// numbers, and overlap on the shared pipeline run dir. Set true before the
// takeover await, cleared once it completes — the rest of the launch is
// synchronous up to status='running', so no concurrent request can interleave.
let _auditLaunching = false;
// Persist Broadcast Live across restarts so the operator's choice survives
// process bounces — without this, every restart silently flips public /live
// back to "paused" even though the admin UI still shows BROADCAST ON.
const BROADCAST_PREF_FILE = path.join(__dirname, 'results', '.broadcast-live');
try { state.broadcastLive = _rfs(BROADCAST_PREF_FILE, 'utf8').trim() === '1'; } catch { state.broadcastLive = false; }
function persistBroadcastPref() {
try { _wfs(BROADCAST_PREF_FILE, state.broadcastLive ? '1' : '0'); } catch {}
}
// Persist SDK choice to disk so it survives restarts
const SDK_PREF_FILE = path.join(__dirname, 'results', '.sdk-preference');
try { state.activeSDK = _rfs(SDK_PREF_FILE, 'utf8').trim() || 'js'; } catch { state.activeSDK = 'js'; }
// Helper: re-hydrate logBuffer from a specific log file on disk. Used both at
// boot (after snapshot restore) and on /api/resume so the SSE init replay
// always carries the in-flight run's full prior log history — not the last
// few lines, and not a different file's contents.
// Parse one on-disk audit line back into a {ts,msg} buffer entry. Lines are
// written as `[<ISO>] <msg>` by audit/pipeline.js logLine. Recover the emission
// timestamp when the prefix is present; lines from older runs (no prefix) yield
// ts:null so the client renders a BLANK time rather than a misleading one.
function parseLogLine(line) {
const m = /^\[(\d{4}-\d\d-\d\dT[\d:.]+Z)\]\s([\s\S]*)$/.exec(line);
if (m) {
const t = Date.parse(m[1]);
return { ts: Number.isNaN(t) ? null : t, msg: m[2] };
}
return { ts: null, msg: line };
}
function hydrateLogBufferFromFile(filePath) {
try {
const txt = _rfs(filePath, 'utf8');
const lines = txt.split('\n').filter(l => l.trim());
const tail = lines.slice(-LOG_BUFFER_MAX);
logBuffer.length = 0;
logBuffer.push(...tail.map(parseLogLine));
return tail.length;
} catch { return 0; }
}
// ─── Test Run Management ─────────────────────────────────────────────────────
import { readFileSync as _rfs, writeFileSync as _wfs, mkdirSync as _mkd, existsSync as _ex, readdirSync as _rd, copyFileSync as _cp, rmSync as _rm, statSync as _statSync, renameSync as _rfs2, appendFileSync as _afs } from 'fs';
const RUNS_DIR = path.join(__dirname, 'results', 'runs');
const RUNS_INDEX = path.join(RUNS_DIR, 'index.json');
if (!_ex(RUNS_DIR)) _mkd(RUNS_DIR, { recursive: true });
function loadRunsIndex() {
if (!_ex(RUNS_INDEX)) return { runs: [], activeRun: null };
try {
return JSON.parse(_rfs(RUNS_INDEX, 'utf8'));
} catch (err) {
console.error(`[loadRunsIndex] corrupt or unreadable ${RUNS_INDEX}: ${err.message}`);
return { runs: [], activeRun: null };
}
}
function saveRunsIndex(index) {
_wfs(RUNS_INDEX, JSON.stringify(index, null, 2), 'utf8');
}
function getNextRunNumber() {
const index = loadRunsIndex();
return (index.runs.length > 0 ? Math.max(...index.runs.map(r => r.number)) : 0) + 1;
}
function saveCurrentRun(label) {
const results = getResults();
if (results.length === 0) return null;
// C-1(b) guard: if the currently-active run is ALREADY a saved run (its index
// entry exists AND aliases the same SQLite dbRunId we'd write to), do NOT
// allocate a new run number / create a duplicate index entry / overwrite the
// SQLite spend downward. This happens after load+retest (or resume-then-Save):
// state.activeDbRunId still points at the saved run. Re-persist into the SAME
// run with accumulateSpend=false so we never reduce stored spend, then return
// the existing number. persistActiveRun keeps index + SQLite in lockstep.
if (state.activeRunNumber != null && state.activeDbRunId != null) {
const idxGuard = loadRunsIndex();
const existing = idxGuard.runs.find(r => r.number === state.activeRunNumber);
if (existing && existing.dbRunId != null && Number(existing.dbRunId) === Number(state.activeDbRunId)) {
// Compare against the CUMULATIVE run spend (runSpentUdvpn), NOT the
// balance-delta spentUdvpn: when resuming an already-saved run, the index
// entry still holds the pre-resume total while the post-resume spend lives
// only in runSpentUdvpn — reading spentUdvpn here would record the stale
// (smaller) figure and silently drop the resume-pass spend from the
// on-chain oracle. accumulateSpend=false treats this as the full cumulative.
const storedSpent = Number(existing.spentUdvpn) || 0;
const liveSpent = Number(state.runSpentUdvpn) || 0;
const passSpent = Math.max(storedSpent, liveSpent);
state.activeRunSaved = true;
try {
const r = persistActiveRun(label || existing.label, { accumulateSpend: false, passSpent });
if (r) return r.number;
} catch (e) {
console.error('[saveCurrentRun] re-persist of active saved run failed:', e.message);
}
// Fall through to a fresh save only if re-persist threw with no result.
return state.activeRunNumber;
}
}
// Persist into the run's RESERVED number (startFreshRun pinned it as
// state.activeRunNumber) — NOT a freshly re-derived getNextRunNumber(). One run
// = one number = one dir = one index entry. Re-deriving here let a concurrent
// delete/save shift max(index.runs) so the snapshot dir, index entry, SQLite
// row, and state.activeRunNumber ended up split across two numbers. Fall back
// to getNextRunNumber only when no run is reserved (e.g. the first-boot
// "Initial Audit" save before activeRunNumber is resolved).
const num = state.activeRunNumber != null ? state.activeRunNumber : getNextRunNumber();
const runDir = path.join(RUNS_DIR, `test-${String(num).padStart(3, '0')}`);
_mkd(runDir, { recursive: true });
// Save results
_wfs(path.join(runDir, 'results.json'), JSON.stringify(results, null, 2), 'utf8');
// Save summary
const passed = results.filter(r => r.actualMbps != null);
const failed = results.filter(r => r.actualMbps == null);
const pass10 = passed.filter(r => r.actualMbps >= 10);
const summary = [
`Test #${num} — ${label || 'Full Audit'}`,
`Date: ${new Date().toISOString()}`,
`${'='.repeat(60)}`,
`Total: ${results.length} | Passed: ${passed.length} | Failed: ${failed.length}`,
`Success Rate: ${results.length > 0 ? (passed.length / results.length * 100).toFixed(1) : '0.0'}%`,
`Pass 10 Mbps SLA: ${pass10.length} (${passed.length > 0 ? (pass10.length / passed.length * 100).toFixed(1) : '0.0'}%)`,
``,
`── Passed Nodes (${passed.length}) ──`,
...passed.map(r => ` ${r.address.slice(0, 25)}... | ${r.actualMbps} Mbps | ${r.type} | ${r.moniker} | ${r.city}, ${r.country} | Google: ${r.googleAccessible === true ? 'YES' : r.googleAccessible === false ? 'NO' : '?'}`),
``,
`── Failed Nodes (${failed.length}) ──`,
...failed.map(r => ` ${r.address.slice(0, 25)}... | ${r.type} | ${r.moniker} | ${r.city}, ${r.country} | peers=${r.peers ?? '?'} | ${(r.error || '').slice(0, 80)}`),
].join('\n');
_wfs(path.join(runDir, 'summary.txt'), summary, 'utf8');
// Copy failures log
const failLog = path.join(__dirname, 'results', 'failures.jsonl');
if (_ex(failLog)) _cp(failLog, path.join(runDir, 'failures.jsonl'));
// Snapshot the run's execution log so loading the run later shows ITS log
// (not the last live run's rolling buffer).
if (state.auditLogPath && _ex(state.auditLogPath)) {
try { _cp(state.auditLogPath, path.join(runDir, 'audit.log')); } catch { }
}
// Update index — find-or-update on the reserved number so re-saving the SAME
// run updates its entry in place instead of pushing a duplicate (one run = one
// entry). spentUdvpn/dbRunId persist the run's net spend + SQLite id so loading
// it later restores Net Spend deterministically; auditLog lets delete purge the
// raw log.
const index = loadRunsIndex();
const entryData = {
number: num,
label: label || 'Full Audit',
date: new Date().toISOString(),
total: results.length,
passed: passed.length,
failed: failed.length,
pass10: pass10.length,
sdk: state.activeSDK,
spentUdvpn: Number(state.runSpentUdvpn) || 0,
dbRunId: state.activeDbRunId || null,
auditLog: state.auditLogPath ? path.basename(state.auditLogPath) : null,
};
const _existingIdx = index.runs.findIndex(r => r.number === num);
if (_existingIdx !== -1) index.runs[_existingIdx] = entryData;
else index.runs.push(entryData);
index.activeRun = num;
saveRunsIndex(index);
// ─── SQLite: mark the run as finished ────────────────────────────────────
if (state.activeDbRunId) {
try {
updateRunOnFinish(state.activeDbRunId, {
finished_at: Date.now(),
node_count: results.length,
pass_count: passed.length,
spent_udvpn: Number(state.runSpentUdvpn) || 0,
});
} catch (dbErr) {
console.error(`[db] updateRunOnFinish failed: ${dbErr.message}`);
}
}
state.activeRunSaved = true;
return num;
}
/**
* Persist the current in-memory results back to the ACTIVE run's snapshot dir
* (test-NNN/) + its index entry, WITHOUT allocating a new run number. Used by
* retest endpoints so a retest's mutated results land on disk + SQLite for the
* same run the operator was viewing — otherwise file / index / SQLite / live
* state diverge (the retest-divergence bug). Re-syncs the index counts in place
* and updates the SQLite run row when one is attached.
*
* Returns { number, cumulativeSpent } written, or null when there's
* nothing/nowhere to save. Callers use cumulativeSpent to reset
* state.spentUdvpn to the true running total (retest passes reset it to
* this-pass-only before this runs).
*
* `passSpent` (optional) is the spend to ACCUMULATE for this pass. Callers
* snapshot state.spentUdvpn at the moment the pass spend is known and pass it
* here, because the idle balance refresher can zero state.spentUdvpn in the gap
* between a retest flipping status to 'done' and this async persist running.
* When omitted, falls back to the live state.spentUdvpn.
*/
function persistActiveRun(label, { accumulateSpend = true, passSpent = null } = {}) {
const num = state.activeRunNumber;
if (num == null) return null;
const results = getResults();
if (results.length === 0) return null;
const runDir = path.join(RUNS_DIR, `test-${String(num).padStart(3, '0')}`);
try { _mkd(runDir, { recursive: true }); } catch (e) { console.error('[persistActiveRun] mkdir failed:', e.message); }
_wfs(path.join(runDir, 'results.json'), JSON.stringify(results, null, 2), 'utf8');
const failLog = path.join(__dirname, 'results', 'failures.jsonl');
if (_ex(failLog)) { try { _cp(failLog, path.join(runDir, 'failures.jsonl')); } catch (e) { console.error('[persistActiveRun] failures copy failed:', e.message); } }
if (state.auditLogPath && _ex(state.auditLogPath)) {
try { _cp(state.auditLogPath, path.join(runDir, 'audit.log')); } catch (e) { console.error('[persistActiveRun] log copy failed:', e.message); }
}
const passed = results.filter(r => r.actualMbps != null);
const failed = results.filter(r => r.actualMbps == null);
const pass10 = passed.filter(r => r.actualMbps >= 10);
// Spend accounting on the retest-persist path:
// runRetestSkips resets state.spentUdvpn to 0 at its top so the
// LIVE header shows ONLY this retest pass's spend. The run's STORED total must
// therefore ACCUMULATE: read the prior cumulative from the existing index
// entry (default 0) and write prior + this-pass to BOTH the index entry and
// the SQLite row. This is specific to persistActiveRun (the no-new-number
// retest path); saveCurrentRun writes a full run's spend and is NOT additive.
// accumulateSpend can be passed false for callers that already hold the full
// cumulative figure in state.
const index = loadRunsIndex();
let entry = index.runs.find(r => r.number === num);
const priorSpent = accumulateSpend ? (Number(entry?.spentUdvpn) || 0) : 0;
// Snapshot the pass spend the caller captured (defends against the idle
// balance refresher zeroing state.spentUdvpn mid-gap); fall back to live state.
const thisPassSpent = passSpent != null ? (Number(passSpent) || 0) : (Number(state.runSpentUdvpn) || 0);
const cumulativeSpent = priorSpent + thisPassSpent;
if (!entry) {
// M-2: no index entry for the active run → create one so index + SQLite
// stay in lockstep instead of diverging (SQLite written below, index left
// stale). prior=0 for a brand-new entry, so cumulative == this pass.
entry = {
number: num,
label: label || 'Full Audit',
date: new Date().toISOString(),
sdk: state.activeSDK || 'js',
auditLog: state.auditLogPath ? path.basename(state.auditLogPath) : null,
};
index.runs.push(entry);
}
entry.total = results.length;
entry.passed = passed.length;
entry.failed = failed.length;
entry.pass10 = pass10.length;
entry.spentUdvpn = cumulativeSpent;
if (label) entry.label = label;
if (state.activeDbRunId) entry.dbRunId = state.activeDbRunId;
saveRunsIndex(index);
if (state.activeDbRunId) {
try {
updateRunOnFinish(state.activeDbRunId, {
finished_at: Date.now(),
node_count: results.length,
pass_count: passed.length,
spent_udvpn: cumulativeSpent,
});
} catch (dbErr) {
console.error(`[persistActiveRun] updateRunOnFinish failed: ${dbErr.message}`);
}
}
state.activeRunSaved = true;
return { number: num, cumulativeSpent };
}
function loadRun(num) {
const runDir = path.join(RUNS_DIR, `test-${String(num).padStart(3, '0')}`);
const resultsPath = path.join(runDir, 'results.json');
if (!_ex(resultsPath)) return null;
return JSON.parse(_rfs(resultsPath, 'utf8'));
}
/**
* Load a saved run's snapshot into the live `state` + working results so the
* dashboard displays it as a read-only historical run. Single source of truth
* for "show this run", used by:
* - POST /api/runs/load/:num (operator picks a run from the dropdown)
* - the boot path (admin always lands on the latest run, never a blank "new")
* - the delete-of-active fallback (drop back to the latest remaining run)
* Returns the loaded results array, or null when the snapshot is missing. Does
* NOT broadcast — callers broadcast/respond as appropriate.
*/
function loadRunIntoState(num) {
const data = loadRun(num);
if (!data) return null;
// Replace the working set with the loaded run.
const results = getResults();
results.length = 0;
results.push(...data);
saveResults(state);
rehydrateState(data);
// A loaded run is a complete set, so its Total is its own node count — not the
// last live audit's chain total (which rehydrateState deliberately leaves in
// place). Without this the header's Total/Remaining stuck on the prior run.
state.totalNodes = data.length;
// Restore this run's spend total (rehydrateState only recomputes per-node
// counts). spentUdvpn is already net.
const _idx = loadRunsIndex();
const _runMeta = _idx.runs.find(r => r.number === num);
let _spent = Number(_runMeta?.spentUdvpn) || 0;
if (_runMeta && _runMeta.spentUdvpn == null) {
// Preferred path: a stored dbRunId makes the spend lookup deterministic.
// Fall back to the getRunSpendByFinish time+count heuristic only when no
// dbRunId was recorded (runs saved before that field existed).
let recovered = false;
if (_runMeta.dbRunId) {
try {
const row = getRun(Number(_runMeta.dbRunId));
if (row) { _spent = Number(row.spent_udvpn) || 0; _runMeta.spentUdvpn = _spent; saveRunsIndex(_idx); recovered = true; }
} catch (e) { console.error('[runs] spend lookup by dbRunId failed:', e.message); }
}
if (!recovered && _runMeta.date) {
try {
const m = getRunSpendByFinish(Date.parse(_runMeta.date), Number(_runMeta.total) || 0);
if (m) { _spent = m.spent_udvpn; _runMeta.spentUdvpn = _spent; saveRunsIndex(_idx); }
} catch (e) { console.error('[runs] spend backfill failed:', e.message); }