Merge pull request #181 from synonymdev/feat/probe-reset-scores

feat: reset pathfinding scores before probes

Author: piotr-iohk

docs/mainnet-probe.md

@@ -18,6 +18,9 @@ It is orchestrated nightly by the private `bitkit-nightly` repo (`mainnet-probe.
 | Variable | Default | Description |
 | --- | --- | --- |
 | `PROBE_ORDER` | `config` | Order of probes per target: `config` = amounts as listed in target config, `desc` = highest amount first (avoids small probes "warming up" scorer knowledge of the route), `random` = global shuffle of all target+amount pairs. |
+| `PROBE_RESET_SCORES` | `false` | When `true`, deletes the persisted pathfinding scores (`scorer` and `external_pathfinding_scores_cache` VSS keys) and restarts the node before probing, so every run starts from a fresh scorer (external scores are re-downloaded on startup). Recommended for scorer A/B experiments; the nightly job enables it by default. Accepts `true/false/1/0/yes/no`. |
+| `PROBE_RESET_SCORES_TIMEOUT_SECONDS` | `180` | Timeout for the scores reset devtools command (covers node stop + VSS deletes + node start). |
+| `PROBE_SCORES_SYNC_MAX_AGE_S` | `900` | Only with `PROBE_RESET_SCORES=true`: readiness additionally requires the node's last external scores sync to be at most this many seconds old **and** newer than the reset floor reported by the app (captured after the node stop + VSS deletes, before the restart; the sync timestamp persisted in node metrics survives the restart, so only a sync from the rebuilt node proves the scores were re-downloaded). Guards against a failed scorer fetch silently producing a "no scores" run. |
 | `PROBE_RETRIES` | `2` | In-test retries per target+amount after a failed probe (total attempts = retries + 1). `0` = single attempt; useful to measure first-attempt success rate. |
 | `PROBE_RETRY_DELAY_MS` | `5000` | Delay between probe retries. |
 | `PROBE_DELAY_MS` | `10000` | Delay between consecutive probes (different target/amount). |
@@ -47,6 +50,7 @@ Probing starts only after the node reports a healthy state (running, peers conne
 | `PROBE_INVOICE_METHOD` | `probeInvoice` | Devtools content-provider method for invoice probes. |
 | `PROBE_NODE_METHOD` | `probeNode` | Devtools method for keysend probes. |
 | `PROBE_READINESS_METHOD` | `probeReadiness` | Devtools method for the readiness check. |
+| `PROBE_RESET_SCORES_METHOD` | `resetScores` | Devtools method for the pathfinding scores reset. |
 
 ### Related (set by orchestration)
 
@@ -95,4 +99,5 @@ Notes:
 
 - The wallet derived from `PROBE_SEED` must already have an open, usable channel with outbound liquidity covering the largest configured probe amount.
 - Probes do not move funds; the wallet balance is unchanged by a run.
-- For scorer experiments, prefer `PROBE_ORDER=desc PROBE_RETRIES=0` to measure cold first-attempt success rate per amount.
+- For scorer experiments, prefer `PROBE_RESET_SCORES=true PROBE_ORDER=desc PROBE_RETRIES=0` to measure cold first-attempt success rate per amount. Without the reset, locally learned scores accumulate in VSS under the probe seed across runs (probe results train the scorer), so consecutive runs are not comparable.
+- `PROBE_RESET_SCORES` requires an app build that includes the `resetScores` devtools method (bitkit-android).

test/helpers/probe.ts

@@ -56,6 +56,8 @@ const DEFAULT_PROBE_FETCH_RETRY_DELAY_MS = 1_000;
 const DEFAULT_READINESS_TIMEOUT_MS = 180_000;
 const DEFAULT_READINESS_POLL_MS = 5_000;
 const DEFAULT_MIN_GRAPH_CHANNELS = 10_000;
+const DEFAULT_RESET_SCORES_TIMEOUT_SECONDS = 180;
+const DEFAULT_SCORES_SYNC_MAX_AGE_S = 900;
 
 export type ProbeReadiness = {
   ready: boolean;
@@ -72,6 +74,7 @@ export type ProbeReadiness = {
   graphNodeCount?: number;
   graphChannelCount?: number;
   latestRgsSyncTimestamp?: number;
+  latestPathfindingScoresSyncTimestamp?: number;
 };
 
 export function resolveProbeTargets(): ProbeTarget[] {
@@ -248,6 +251,75 @@ export function summarizeProbeCommandFailure(raw: string): string {
   );
 }
 
+export function resolveProbeResetScores(): boolean {
+  return parseBooleanEnv('PROBE_RESET_SCORES') ?? false;
+}
+
+/**
+ * Resets the persisted pathfinding scores via devtools and returns the
+ * device-clock epoch seconds to be used as a floor for the scores sync
+ * timestamp in readiness checks (the sync timestamp persisted in node metrics
+ * survives the restart, so only a sync strictly newer than the reset proves
+ * the external scores were re-downloaded). The app reports the floor as the
+ * moment after the node stop + VSS deletes and before the restart, so any
+ * newer sync can only come from the rebuilt node; if the app is too old to
+ * report it, falls back to the device time captured before the reset call.
+ * The floor uses the device clock because the sync timestamp is also
+ * device-generated, making the comparison immune to host/device clock skew.
+ */
+export async function resetPathfindingScores({ logPrefix }: { logPrefix: string }): Promise<number> {
+  const method = process.env.PROBE_RESET_SCORES_METHOD ?? 'resetScores';
+  const timeoutSeconds =
+    parsePositiveIntEnv('PROBE_RESET_SCORES_TIMEOUT_SECONDS') ?? DEFAULT_RESET_SCORES_TIMEOUT_SECONDS;
+
+  console.info(`→ [${logPrefix}] Resetting pathfinding scores (timeout ${timeoutSeconds}s)...`);
+  const fallbackFloorS = getDeviceEpochSeconds();
+  const raw = runDevToolsCommand(method, {}, timeoutSeconds);
+  if (!parseProbeCommandSuccess(raw)) {
+    throw new Error(`Pathfinding scores reset failed: ${summarizeProbeCommandFailure(raw)}`);
+  }
+  const deviceResetAtS = parseResetTimestamp(raw);
+  if (deviceResetAtS === null) {
+    console.warn(
+      `→ [${logPrefix}] Reset result has no timestamp (old app build?); using pre-reset device time as scores sync floor`
+    );
+  }
+  const resetFloorS = deviceResetAtS ?? fallbackFloorS;
+  console.info(`→ [${logPrefix}] Pathfinding scores reset done (floor ${resetFloorS})`);
+  return resetFloorS;
+}
+
+function parseResetTimestamp(raw: string): number | null {
+  const result = extractContentCallResult(raw);
+  if (!result) return null;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(result);
+  } catch {
+    return null;
+  }
+  if (typeof parsed !== 'object' || parsed === null) return null;
+  if (!('timestamp' in parsed)) return null;
+
+  const timestamp = parsed.timestamp;
+  return typeof timestamp === 'number' && Number.isFinite(timestamp) && timestamp > 0
+    ? timestamp
+    : null;
+}
+
+function getDeviceEpochSeconds(): number {
+  const raw = execFileSync('adb', ['shell', 'date', '+%s'], {
+    encoding: 'utf8',
+    timeout: 10_000,
+  });
+  const epoch = Number.parseInt(raw.trim(), 10);
+  if (!Number.isFinite(epoch) || epoch <= 0) {
+    throw new Error(`Failed to read device epoch time: ${raw.trim() || 'empty output'}`);
+  }
+  return epoch;
+}
+
 export function runReadinessCommand(): string {
   const method = process.env.PROBE_READINESS_METHOD ?? 'probeReadiness';
   const command = [
@@ -305,18 +377,39 @@ function summarizeReadinessError(raw: string): string {
 
 export function isProbeReadinessSufficient(
   readiness: ProbeReadiness,
-  minGraphChannels: number
+  minGraphChannels: number,
+  maxScoresSyncAgeS: number | null = null,
+  minScoresSyncTimestamp: number | null = null,
+  nowS: number = Date.now() / 1000
 ): boolean {
   return (
     readiness.ready &&
     readiness.nodeRunning &&
     readiness.connectedPeers > 0 &&
     readiness.usableChannels > 0 &&
     readiness.syncHealthy &&
-    (readiness.graphChannelCount ?? 0) >= minGraphChannels
+    (readiness.graphChannelCount ?? 0) >= minGraphChannels &&
+    isScoresSyncFresh(readiness, maxScoresSyncAgeS, minScoresSyncTimestamp, nowS)
   );
 }
 
+function isScoresSyncFresh(
+  readiness: ProbeReadiness,
+  maxAgeS: number | null,
+  minTimestamp: number | null,
+  nowS: number = Date.now() / 1000
+): boolean {
+  if (maxAgeS === null) return true;
+  const timestamp = readiness.latestPathfindingScoresSyncTimestamp;
+  if (!timestamp) return false;
+  if (nowS - timestamp > maxAgeS) return false;
+  // Both timestamps come from the device clock; the floor is captured by the
+  // app after the node stop + VSS deletes, so any strictly newer sync can
+  // only come from the rebuilt node (post-reset).
+  if (minTimestamp !== null && timestamp <= minTimestamp) return false;
+  return true;
+}
+
 export function summarizeProbeReadiness(readiness: ProbeReadiness): string {
   return [
     `running=${readiness.nodeRunning}`,
@@ -325,25 +418,35 @@ export function summarizeProbeReadiness(readiness: ProbeReadiness): string {
     `outboundSats=${readiness.outboundCapacitySats}`,
     `graphChannels=${readiness.graphChannelCount ?? 'n/a'}`,
     `graphNodes=${readiness.graphNodeCount ?? 'n/a'}`,
+    `scoresSync=${readiness.latestPathfindingScoresSyncTimestamp ?? 'n/a'}`,
     `syncHealthy=${readiness.syncHealthy}`,
     `ready=${readiness.ready}`,
   ].join(' ');
 }
 
 type WaitForProbeReadinessOptions = {
   logPrefix: string;
+  requireScoresSync?: boolean;
+  /** Device-clock epoch seconds; scores sync must be strictly newer than this (the reset floor reported by the app). */
+  minScoresSyncTimestamp?: number | null;
 };
 
 export async function waitForProbeReadiness({
   logPrefix,
+  requireScoresSync = false,
+  minScoresSyncTimestamp = null,
 }: WaitForProbeReadinessOptions): Promise<ProbeReadiness> {
   const timeoutMs = parsePositiveIntEnv('PROBE_READINESS_TIMEOUT_MS') ?? DEFAULT_READINESS_TIMEOUT_MS;
   const pollMs = parsePositiveIntEnv('PROBE_READINESS_POLL_MS') ?? DEFAULT_READINESS_POLL_MS;
   const minGraphChannels =
     parseNonNegativeIntEnv('PROBE_MIN_GRAPH_CHANNELS') ?? DEFAULT_MIN_GRAPH_CHANNELS;
+  const maxScoresSyncAgeS = requireScoresSync
+    ? (parsePositiveIntEnv('PROBE_SCORES_SYNC_MAX_AGE_S') ?? DEFAULT_SCORES_SYNC_MAX_AGE_S)
+    : null;
+  const minSyncTimestamp = requireScoresSync ? minScoresSyncTimestamp : null;
 
   console.info(
-    `→ [${logPrefix}] Waiting for probe readiness (timeout ${timeoutMs / 1000}s, minGraphChannels ${minGraphChannels})...`
+    `→ [${logPrefix}] Waiting for probe readiness (timeout ${timeoutMs / 1000}s, minGraphChannels ${minGraphChannels}, requireScoresSync ${requireScoresSync})...`
   );
 
   const deadline = Date.now() + timeoutMs;
@@ -360,7 +463,10 @@ export async function waitForProbeReadiness({
     const readiness = raw ? parseProbeReadiness(raw) : null;
     if (readiness) {
       lastSummary = summarizeProbeReadiness(readiness);
-      if (isProbeReadinessSufficient(readiness, minGraphChannels)) {
+      // Use the device clock for the scores sync age check so it is measured
+      // against the same clock that produced the sync timestamp.
+      const nowS = maxScoresSyncAgeS !== null ? getDeviceEpochSeconds() : Date.now() / 1000;
+      if (isProbeReadinessSufficient(readiness, minGraphChannels, maxScoresSyncAgeS, minSyncTimestamp, nowS)) {
         console.info(`→ [${logPrefix}] Probe readiness satisfied: ${lastSummary}`);
         return readiness;
       }
@@ -409,6 +515,7 @@ export function renderProbeReport(
     '',
     `Required failures: ${failedRequired.length}`,
     `Probe order: ${probeOrderForReport()}`,
+    `Scores reset: ${scoresResetForReport()}`,
     `Readiness at probe start: ${readiness ? summarizeProbeReadiness(readiness) : 'not captured'}`,
     '',
     '| Target | Type | Amount sats | Required | Fetch | Probe | Retries | Duration ms | Failure |',
@@ -444,6 +551,14 @@ function probeOrderForReport(): string {
   }
 }
 
+function scoresResetForReport(): string {
+  try {
+    return String(resolveProbeResetScores());
+  } catch {
+    return `invalid (${process.env.PROBE_RESET_SCORES})`;
+  }
+}
+
 function parseProbeTarget(value: unknown): ProbeTarget {
   if (typeof value !== 'object' || value === null) {
     throw new Error('Each probe target must be an object');
@@ -560,6 +675,14 @@ export function parseNonNegativeIntEnv(name: string): number | null {
   return value;
 }
 
+function parseBooleanEnv(name: string): boolean | null {
+  const raw = process.env[name]?.trim().toLowerCase();
+  if (!raw) return null;
+  if (raw === 'true' || raw === '1' || raw === 'yes') return true;
+  if (raw === 'false' || raw === '0' || raw === 'no') return false;
+  throw new Error(`Invalid ${name} value: ${raw} (expected true or false)`);
+}
+
 function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }

test/specs/mainnet/probe.e2e.ts

@@ -6,7 +6,9 @@ import {
   parseNonNegativeIntEnv,
   parseProbeCommandSuccess,
   probeModeForTargetType,
+  resetPathfindingScores,
   resolveProbeOrder,
+  resolveProbeResetScores,
   resolveProbeTargets,
   runProbeInvoiceCommand,
   runProbeNodeCommand,
@@ -217,7 +219,17 @@ describe('@probe_mainnet - Lightning probe smoke', () => {
         expectAndroidAlert: false,
       });
       await waitForMainnetWalletReady({ logPrefix: 'Probe' });
-      readiness = await waitForProbeReadiness({ logPrefix: 'Probe' });
+
+      const resetScores = resolveProbeResetScores();
+      let scoresResetFloorS: number | null = null;
+      if (resetScores) {
+        scoresResetFloorS = await resetPathfindingScores({ logPrefix: 'Probe' });
+      }
+      readiness = await waitForProbeReadiness({
+        logPrefix: 'Probe',
+        requireScoresSync: resetScores,
+        minScoresSyncTimestamp: scoresResetFloorS,
+      });
 
       const probeOrder = resolveProbeOrder();
       const probes = buildProbeQueue(targets, probeOrder);