fix(eta): retest-aware + skew-proof remaining-duration wire + loop-staleness refresh + explicit ring reset

Author: Sentinel-Bluebuilder

admin.html

@@ -1324,7 +1324,7 @@ <h1 style="margin:0;flex:0 0 auto">SENTINEL NODE TEST</h1>
     onReady(() => {
       // Load stats instantly (tiny payload, no results)
       fetch('/api/stats').then(r => r.json()).then(d => {
-        if (d.state) { state = Object.assign(state, d.state); applyState(); }
+        if (d.state) { state = Object.assign(state, d.state); anchorEta(); applyState(); }
       }).catch(() => {});
       loadSdkVersions();
       connectSSE();
@@ -1449,6 +1449,7 @@ <h1 style="margin:0;flex:0 0 auto">SENTINEL NODE TEST</h1>
         const msg = JSON.parse(e.data);
         if (msg.type === 'init' || msg.type === 'state') {
           state = Object.assign(state, msg.state);
+          anchorEta(); // re-anchor the skew-immune ETA on every state update
           if (msg.results) { resultsArr = msg.results; renderTable(); }
           // Restore logs from server buffer on init — batch-render in a single
           // DocumentFragment so a full 5000-line buffer doesn't trigger one
@@ -1469,9 +1470,10 @@ <h1 style="margin:0;flex:0 0 auto">SENTINEL NODE TEST</h1>
           }
           applyState();
         } else if (msg.type === 'progress') {
-          state = Object.assign(state, msg.state); applyState();
+          state = Object.assign(state, msg.state); anchorEta(); applyState();
         } else if (msg.type === 'result') {
           state = Object.assign(state, msg.state);
+          anchorEta(); // result events carry fresh counters → fresh etaRemainingMs
           if (msg.result) { upsertLocal(msg.result); addSingleRow(msg.result); }
           applyState();
         } else if (msg.type === 'log') {
@@ -1487,16 +1489,30 @@ <h1 style="margin:0;flex:0 0 auto">SENTINEL NODE TEST</h1>
       };
     }
 
+    // Skew-immune ETA anchor. The server broadcasts a REMAINING DURATION
+    // (state.etaRemainingMs), not an absolute epoch. We snapshot it against THIS
+    // client's own clock the instant a state update lands, then the 1s ticker
+    // counts it down using only our clock — so a skewed browser clock can't
+    // distort the ETA. A new server value RE-ANCHORS (server corrections land);
+    // between updates we just tick our anchor down. null clears the anchor.
+    let _etaAnchor = null; // { remainingMs, at } or null
+    function anchorEta() {
+      if (Number.isFinite(state.etaRemainingMs)) {
+        _etaAnchor = { remainingMs: state.etaRemainingMs, at: Date.now() };
+      } else {
+        _etaAnchor = null;
+      }
+    }
+
     function updateETA() {
-      // Server-authoritative ETA: the server computes ONE absolute finish epoch
-      // (state.etaFinishAtMs) from a windowed real-throughput estimate and
-      // broadcasts it. Both admin and live render the same value — we just
-      // count down to it. See computeEtaFinishAt() in server.js.
+      // Render the anchored remaining duration, decremented by the time elapsed
+      // on OUR clock since the anchor was set. See computeEtaRemainingMs() +
+      // the etaRemainingMs broadcast in server.js.
       const el = document.getElementById('etaTime');
       if (state.status === 'done') { el.textContent = '00:00:00'; return; }
       if (state.status !== 'running') return;
-      if (Number.isFinite(state.etaFinishAtMs)) {
-        const rem = Math.max(0, state.etaFinishAtMs - Date.now());
+      if (_etaAnchor) {
+        const rem = Math.max(0, _etaAnchor.remainingMs - (Date.now() - _etaAnchor.at));
         const h = Math.floor(rem / 3600000), m = Math.floor((rem % 3600000) / 60000), s = Math.floor((rem % 60000) / 1000);
         el.textContent = `${String(h).padStart(2,'0')}:${String(m).padStart(2,'0')}:${String(s).padStart(2,'0')}`;
       } else {

audit/continuous.js

@@ -303,6 +303,21 @@ async function _runOnePass(loopState, batchId, frozenNodes = null, resume = fals
     _emitScoped('result', { result: raw, batchId });
     const payload = _sanitizeBatchNodeResult(raw, batchId);
     _emitScoped('batch:node:result', payload);
+    // ETA staleness fix: the pipeline only emits a per-node `state` event on
+    // SUCCESS (it gates that broadcast on actualMbps != null — pipeline.js:978),
+    // and batchBroadcast forwards that success `state` via the type==='state'
+    // branch above. During a failing streak no `state` reaches the bus and the
+    // server's windowed ETA (etaRemainingMs) never refreshes — clients tick
+    // their anchor down to 0 and sit there. So forward the loop's current state
+    // (carried on the result payload, with up-to-date counters) as a lightweight
+    // `state` event ONLY for nodes the pipeline skipped its state emit on
+    // (actualMbps == null: failures + skips). Successful nodes already got their
+    // state above; emitting again here would double every success frame. Net:
+    // exactly one `state` per completed node, pass/fail alike — and the server
+    // recomputes etaRemainingMs each time. No new fields, no public-payload bloat.
+    if (data.state && typeof data.state === 'object' && raw.actualMbps == null) {
+      _emitScoped('state', { state: data.state });
+    }
     // Persist to batch_results (non-blocking, non-fatal). Guard on batchId > 0:
     // if insertBatch failed upstream (logged, non-fatal) currentBatchId stays 0,
     // and writing insertBatchResult(0, …) would violate the FK to a non-existent

live.html

@@ -1650,9 +1650,9 @@ <h1 class="live-page-title">
       if (hdrDot) hdrDot.className = 'pulse-dot';
       // Log-panel header label: 'stopped' / 'complete' / 'error' / 'idle'.
       setStatus(status === 'done' ? 'complete' : status);
-      // cbRender recomputes the ETA: with startedAt now null it resolves to
-      // "ETA 00:00:00" (all tested) or "ETA —" (stopped mid-run), never a
-      // climbing number.
+      // cbRender recomputes the ETA: a 'done' run reads "ETA 00:00:00" (matches
+      // admin), any other terminal/stopped status with no anchor reads "ETA —",
+      // never a climbing number.
       cbRender();
     }
 
@@ -1718,14 +1718,23 @@ <h1 class="live-page-title">
 
       const etaEl = _cbEl('cbEta');
       if (etaEl) {
-        // Server-authoritative ETA: the server computes ONE absolute finish
-        // epoch (_liveState.etaFinishAtMs) from a windowed real-throughput
-        // estimate and broadcasts it. We just count down to it — so this page
-        // and admin.html always show the same value. The old elapsed/tested
-        // per-node math (which diverged from admin's) is gone.
-        const finishAt = Number(_liveState && _liveState.etaFinishAtMs);
-        if (Number.isFinite(finishAt) && finishAt > 0) {
-          const rem = Math.max(0, finishAt - Date.now());
+        // Skew-immune ETA: the server broadcasts a REMAINING DURATION
+        // (_liveState.etaRemainingMs), not an absolute epoch. We anchor it to
+        // THIS client's own clock at receipt (see mergeLiveState → _etaAnchor)
+        // and count it down using only our clock — so a skewed browser clock
+        // can't distort the ETA. This page and admin.html show the same value.
+        // Parity with admin.html (updateETA): a terminal 'done' run shows a
+        // zeroed clock — "ETA 00:00:00" — NOT "ETA —". The server stops
+        // broadcasting a positive etaRemainingMs once status leaves 'running'
+        // (computeEtaRemainingMs returns null), so _etaAnchor is already cleared
+        // by the time a run reads 'done'; without this branch a completed run
+        // would show "ETA —" here while admin shows "00:00:00". "ETA —" stays
+        // reserved for genuinely unknown / stopped-mid-run states (no anchor,
+        // not done).
+        if (_liveState && _liveState.status === 'done') {
+          etaEl.textContent = 'ETA 00:00:00';
+        } else if (_etaAnchor) {
+          const rem = Math.max(0, _etaAnchor.remainingMs - (Date.now() - _etaAnchor.at));
           const h = Math.floor(rem / 3600000);
           const m = Math.floor((rem % 3600000) / 60000);
           const s = Math.floor((rem % 60000) / 1000);
@@ -1890,11 +1899,11 @@ <h1 class="live-page-title">
       applyHeaderStatsFromState();
     }, 5000);
 
-    // Dedicated 1s ETA ticker. The ETA is now a countdown to the
-    // server-broadcast absolute finish epoch (_liveState.etaFinishAtMs), so it
-    // must re-render every second to tick down between server events. Gated on
-    // _cb.startedAt (cbFreeze nulls it on stop/done/error/idle) so a finished
-    // run's ETA freezes instead of marching.
+    // Dedicated 1s ETA ticker. The ETA is a countdown of the server-broadcast
+    // REMAINING DURATION (_etaAnchor, anchored to our own clock in
+    // mergeLiveState), so it must re-render every second to tick the anchor down
+    // between server events. Gated on _cb.startedAt (cbFreeze nulls it on
+    // stop/done/error/idle) so a finished run's ETA freezes instead of marching.
     setInterval(() => {
       if (_cb.startedAt) cbRender();
     }, 1000);
@@ -1914,6 +1923,17 @@ <h1 class="live-page-title">
     const LIVE_CACHE_KEY = 'live:snapshot:v1';
     const LIVE_CACHE_TTL_MS = 60 * 60 * 1000; // 1h — enough to survive a reload
     let _liveState = {};
+    // Skew-immune ETA anchor. The server broadcasts a REMAINING DURATION
+    // (_liveState.etaRemainingMs); we snapshot it against THIS client's own
+    // clock the instant a state update lands (mergeLiveState), then cbRender's
+    // 1s ticker counts it down using only our clock. A new server value
+    // re-anchors; null clears it (cbRender shows "ETA —").
+    let _etaAnchor = null; // { remainingMs, at } or null
+    function anchorEta() {
+      const rem = Number(_liveState && _liveState.etaRemainingMs);
+      if (Number.isFinite(rem)) _etaAnchor = { remainingMs: rem, at: Date.now() };
+      else _etaAnchor = null;
+    }
     // True once we've received an AUTHORITATIVE run status — either the REST
     // live-state seed or an SSE state/init. Until then, applyPauseFromState must
     // NOT decide "paused" from the empty/cached defaults: on a cold load the
@@ -1988,6 +2008,7 @@ <h1 class="live-page-title">
       // state/init) — applyPauseFromState may now decide the overlay.
       _liveStateResolved = true;
       _liveState = { ..._liveState, ...s };
+      anchorEta(); // re-anchor the skew-immune ETA on every state update
       applyHeaderStatsFromState();
       applyPauseFromState();
       schedulePersist();
@@ -2448,8 +2469,9 @@ <h1 class="live-page-title">
               // paused_balance / paused_internet — show a friendly 'paused' label.
               // Not terminal: don't cbFreeze (keep startedAt so the resumed run's
               // ETA continues). The ETA is frozen meanwhile by the server:
-              // computeEtaFinishAt returns null when status !== 'running', so the
-              // broadcast etaFinishAtMs goes null and cbRender shows "ETA —".
+              // computeEtaRemainingMs returns null when status !== 'running', so
+              // the broadcast etaRemainingMs goes null, anchorEta clears
+              // _etaAnchor, and cbRender shows "ETA —".
               setStatus('paused');
             }
             renderLiveStats();

server.js

@@ -353,29 +353,56 @@ function appendEventLog(msg) {
 // different inputs, so they disagreed. The server now owns the single source
 // of truth: it keeps a rolling window of the last ETA_WINDOW node-completion
 // timestamps, derives the current completion RATE (nodes/ms), and broadcasts
-// an ABSOLUTE finish epoch (`etaFinishAtMs`). Clients render
-// max(0, etaFinishAtMs - Date.now()) as HH:MM:SS, ticking every second. This
+// a REMAINING DURATION (`etaRemainingMs`) — NOT an absolute epoch. Each client
+// anchors that duration to its OWN clock at receipt and counts it down, so a
+// skewed browser clock can't distort the ETA (an absolute server epoch rendered
+// against a skewed client Date.now() drifted by the offset — minutes). The math
 // is throughput-based (parallelism-aware), not per-node-duration based.
 const ETA_WINDOW = 12;
 let _etaCompletions = [];   // recent node-completion epoch ms (rolling, current pass)
 let _etaLastDone = -1;      // last seen completed-count, to detect a new pass
+let _etaPrevStatus = null;  // last seen status, to detect a fresh →running transition
+
+// Resolve the (done, total) pair the ETA should measure against. During a
+// retest (runRetestSkips), every node already has a result row so the whole-run
+// counters are saturated (done ≈ total) and remaining ≤ 0 — the ETA would pin
+// at 00:00:00 for the entire retest. The retest's REAL progress lives in
+// separate fields (retestTested / retestPassed+retestFailed / retestTotal), so
+// when retestMode is set with a usable retestTotal we measure against those
+// instead. Falls back to the whole-run counters whenever the retest fields are
+// absent or non-positive (defensive — never let a missing field zero the ETA).
+function etaProgress(st) {
+  if (st.retestMode && Number(st.retestTotal) > 0) {
+    // retestTested already counts every retested node (pass + fail), so it is
+    // the authoritative "done" for the retest pass; fall back to summing
+    // retestPassed+retestFailed if retestTested is somehow absent.
+    let done = Number(st.retestTested);
+    if (!(done >= 0)) done = (Number(st.retestPassed) || 0) + (Number(st.retestFailed) || 0);
+    return { done, total: Number(st.retestTotal) };
+  }
+  const done = (st.testedNodes || 0) + (st.failedNodes || 0) + (st.skippedNodes || 0);
+  const total = st.totalNodes || 0;
+  return { done, total };
+}
 
-function computeEtaFinishAt(st) {
+// Returns the REMAINING milliseconds until the active pass completes, or null
+// when not computable (not running / <2 completions / total<=0). 0 when there
+// is no work left. Clients anchor this to their own clock and tick it down.
+function computeEtaRemainingMs(st) {
   if (!st || typeof st !== 'object') return null;
   // Only meaningful while a pass is actively running. done/idle/paused_* → null.
   if (st.status !== 'running') return null;
-  const done = (st.testedNodes || 0) + (st.failedNodes || 0) + (st.skippedNodes || 0);
-  const total = st.totalNodes || 0;
+  const { done, total } = etaProgress(st);
   if (total <= 0) return null;
   const remaining = total - done;
-  if (remaining <= 0) return Date.now(); // finish now → renders 00:00:00
+  if (remaining <= 0) return 0; // no work left → renders 00:00:00
   if (_etaCompletions.length < 2) return null; // not enough data → client shows "Calculating…"
   const span = _etaCompletions[_etaCompletions.length - 1] - _etaCompletions[0];
   if (span <= 0) return null;
   const n = _etaCompletions.length - 1; // intervals across the window
   const ratePerMs = n / span;
   const etaMs = remaining / ratePerMs;
-  return Date.now() + Math.round(etaMs);
+  return Math.round(etaMs);
 }
 
 function broadcast(type, data = {}) {
@@ -396,9 +423,9 @@ function broadcast(type, data = {}) {
   if (type === 'state' || type === 'result') saveStateSnapshot();
   // ─── ETA bookkeeping ──────────────────────────────────────────────────────
   // Record the just-finished node FIRST (so it's in the window), then compute
-  // the absolute finish epoch and stamp it onto any state payload so admin
-  // (which ticks on 'result' events) and live (which ticks on 'state') both
-  // render the same value.
+  // the remaining duration and stamp it onto any state payload so admin (which
+  // ticks on 'result' events) and live (which ticks on 'state') both anchor the
+  // same value to their own clocks.
   if (type === 'result') {
     _etaCompletions.push(Date.now());
     if (_etaCompletions.length > ETA_WINDOW) {
@@ -407,16 +434,28 @@ function broadcast(type, data = {}) {
   }
   if (data && data.state && typeof data.state === 'object') {
     const s = data.state;
-    const done = (s.testedNodes || 0) + (s.failedNodes || 0) + (s.skippedNodes || 0);
-    if (done < _etaLastDone) _etaCompletions = []; // counters reset → new pass
+    // Explicit fresh-run ring reset: when status transitions INTO 'running' from
+    // any non-running status, wipe the window so a new run can never inherit the
+    // prior run's completion timestamps. This no longer relies on an incidental
+    // zero-`done` broadcast landing before the first result of the new run.
+    if (s.status === 'running' && _etaPrevStatus !== 'running') {
+      _etaCompletions = [];
+      _etaLastDone = -1;
+    }
+    _etaPrevStatus = s.status;
+    const { done } = etaProgress(s);
+    // Belt-and-suspenders per-pass reset within a continuous loop: each
+    // iteration's counters reset to 0, so a drop below the last-seen done count
+    // means a new pass started — clear the window.
+    if (done < _etaLastDone) _etaCompletions = [];
     _etaLastDone = done;
     // NOTE: for most callers data.state IS the long-lived global `state` object,
-    // so this assignment MUTATES the global in place — etaFinishAtMs is not a
+    // so this assignment MUTATES the global in place — etaRemainingMs is not a
     // per-payload-only field. That's fine: it's recomputed on every broadcast,
-    // and computeEtaFinishAt SELF-CLEARS it to null whenever status !== 'running'
-    // (done/idle/paused_*), so a stale value can never linger on the global.
-    // saveStateSnapshot's allowlist also excludes it, so it never persists.
-    data.state.etaFinishAtMs = computeEtaFinishAt(data.state);
+    // and computeEtaRemainingMs SELF-CLEARS it to null whenever status !==
+    // 'running' (done/idle/paused_*), so a stale value can never linger on the
+    // global. saveStateSnapshot's allowlist also excludes it, so it never persists.
+    data.state.etaRemainingMs = computeEtaRemainingMs(data.state);
   }
   // NOTE: spread `data` FIRST so a payload field named `type` (e.g. the node's
   // service-type like 'wireguard') cannot clobber the SSE event type. The
@@ -1954,13 +1993,13 @@ function _redactPublicError(v, max = 200) {
 //     the Server-Baseline header tile.
 //   - activeSDK ('js' | 'tkd' | 'csharp') surfaces next to the run-mode label;
 //     display name + version are joined client-side via /api/public/sdk-info.
-//   - etaFinishAtMs is the server-authoritative absolute finish epoch (ms);
-//     live.html renders max(0, etaFinishAtMs - Date.now()) as a ticking ETA so
-//     it matches admin.html exactly.
+//   - etaRemainingMs is the server-authoritative REMAINING duration (ms);
+//     live.html anchors it to its own clock at receipt and counts it down, so it
+//     matches admin.html without being distorted by client clock skew.
 const PUBLIC_STATE_KEYS = [
   'status',
   'totalNodes',
-  'etaFinishAtMs',
+  'etaRemainingMs',
   'baselineMbps',
   'baselineHistory',
   'testRun',
@@ -2374,10 +2413,10 @@ app.get('/api/events', adminOnly, rlAdminSse, (req, res) => {
   // Strip wallet + balance internals AND runGranter (subscription granter
   // address — operator-internal, never needs to leave the server).
   const { walletAddress, balance, balanceUdvpn, spentUdvpn, runGranter, ...stateForSse } = state;
-  // The global `state` doesn't carry a fresh etaFinishAtMs (it's stamped onto
+  // The global `state` doesn't carry a fresh etaRemainingMs (it's stamped onto
   // broadcast payloads, not the global), so a freshly-connected admin would be
   // blank until the next event. Compute it once here for the init frame.
-  stateForSse.etaFinishAtMs = computeEtaFinishAt(state);
+  stateForSse.etaRemainingMs = computeEtaRemainingMs(state);
   // Trim each result row's diag to the 4 fields the dashboard reads (drop the
   // credential/config/stdout blob). New array of clones — getResults() returns
   // the shared in-memory state rows; never mutate them.