feat: reset pathfinding scores before probes

docs/mainnet-probe.md

@@ -18,6 +18,9 @@ It is orchestrated nightly by the private `bitkit-nightly` repo (`mainnet-probe.
 | Variable | Default | Description |
 | --- | --- | --- |
 | `PROBE_ORDER` | `config` | Order of probes per target: `config` = amounts as listed in target config, `desc` = highest amount first (avoids small probes "warming up" scorer knowledge of the route), `random` = global shuffle of all target+amount pairs. |
+| `PROBE_RESET_SCORES` | `false` | When `true`, deletes the persisted pathfinding scores (`scorer` and `external_pathfinding_scores_cache` VSS keys) and restarts the node before probing, so every run starts from a fresh scorer (external scores are re-downloaded on startup). Recommended for scorer A/B experiments; the nightly job enables it by default. Accepts `true/false/1/0/yes/no`. |
+| `PROBE_RESET_SCORES_TIMEOUT_SECONDS` | `180` | Timeout for the scores reset devtools command (covers node stop + VSS deletes + node start). |
+| `PROBE_SCORES_SYNC_MAX_AGE_S` | `900` | Only with `PROBE_RESET_SCORES=true`: readiness additionally requires the node's last external scores sync to be at most this many seconds old **and** newer than the reset start time (the sync timestamp persisted in node metrics survives the restart, so only a post-reset sync proves the scores were re-downloaded). Guards against a failed scorer fetch silently producing a "no scores" run. |
 | `PROBE_RETRIES` | `2` | In-test retries per target+amount after a failed probe (total attempts = retries + 1). `0` = single attempt; useful to measure first-attempt success rate. |
 | `PROBE_RETRY_DELAY_MS` | `5000` | Delay between probe retries. |
 | `PROBE_DELAY_MS` | `10000` | Delay between consecutive probes (different target/amount). |
@@ -47,6 +50,7 @@ Probing starts only after the node reports a healthy state (running, peers conne
 | `PROBE_INVOICE_METHOD` | `probeInvoice` | Devtools content-provider method for invoice probes. |
 | `PROBE_NODE_METHOD` | `probeNode` | Devtools method for keysend probes. |
 | `PROBE_READINESS_METHOD` | `probeReadiness` | Devtools method for the readiness check. |
+| `PROBE_RESET_SCORES_METHOD` | `resetScores` | Devtools method for the pathfinding scores reset. |
 
 ### Related (set by orchestration)
 
@@ -95,4 +99,5 @@ Notes:
 
 - The wallet derived from `PROBE_SEED` must already have an open, usable channel with outbound liquidity covering the largest configured probe amount.
 - Probes do not move funds; the wallet balance is unchanged by a run.
-- For scorer experiments, prefer `PROBE_ORDER=desc PROBE_RETRIES=0` to measure cold first-attempt success rate per amount.
+- For scorer experiments, prefer `PROBE_RESET_SCORES=true PROBE_ORDER=desc PROBE_RETRIES=0` to measure cold first-attempt success rate per amount. Without the reset, locally learned scores accumulate in VSS under the probe seed across runs (probe results train the scorer), so consecutive runs are not comparable.
+- `PROBE_RESET_SCORES` requires an app build that includes the `resetScores` devtools method (bitkit-android).

test/helpers/probe.ts

@@ -56,6 +56,8 @@ const DEFAULT_PROBE_FETCH_RETRY_DELAY_MS = 1_000;
 const DEFAULT_READINESS_TIMEOUT_MS = 180_000;
 const DEFAULT_READINESS_POLL_MS = 5_000;
 const DEFAULT_MIN_GRAPH_CHANNELS = 10_000;
+const DEFAULT_RESET_SCORES_TIMEOUT_SECONDS = 180;
+const DEFAULT_SCORES_SYNC_MAX_AGE_S = 900;
 
 export type ProbeReadiness = {
   ready: boolean;
@@ -72,6 +74,7 @@ export type ProbeReadiness = {
   graphNodeCount?: number;
   graphChannelCount?: number;
   latestRgsSyncTimestamp?: number;
+  latestPathfindingScoresSyncTimestamp?: number;
 };
 
 export function resolveProbeTargets(): ProbeTarget[] {
@@ -248,6 +251,46 @@ export function summarizeProbeCommandFailure(raw: string): string {
   );
 }
 
+export function resolveProbeResetScores(): boolean {
+  return parseBooleanEnv('PROBE_RESET_SCORES') ?? false;
+}
+
+/**
+ * Resets the persisted pathfinding scores via devtools and returns the
+ * device-clock epoch seconds at which the reset started, to be used as a
+ * floor for the scores sync timestamp in readiness checks (the sync timestamp
+ * persisted in node metrics survives the restart, so only a sync strictly
+ * newer than the reset proves the external scores were re-downloaded). The
+ * floor is read from the device clock because the sync timestamp is also
+ * device-generated, making the comparison immune to host/device clock skew.
+ */
+export async function resetPathfindingScores({ logPrefix }: { logPrefix: string }): Promise<number> {
+  const method = process.env.PROBE_RESET_SCORES_METHOD ?? 'resetScores';
+  const timeoutSeconds =
+    parsePositiveIntEnv('PROBE_RESET_SCORES_TIMEOUT_SECONDS') ?? DEFAULT_RESET_SCORES_TIMEOUT_SECONDS;
+
+  console.info(`→ [${logPrefix}] Resetting pathfinding scores (timeout ${timeoutSeconds}s)...`);
+  const resetStartedAtS = getDeviceEpochSeconds();
+  const raw = runDevToolsCommand(method, {}, timeoutSeconds);
+  if (!parseProbeCommandSuccess(raw)) {
+    throw new Error(`Pathfinding scores reset failed: ${summarizeProbeCommandFailure(raw)}`);
+  }
+  console.info(`→ [${logPrefix}] Pathfinding scores reset done (started at ${resetStartedAtS})`);
+  return resetStartedAtS;
+}
+
+function getDeviceEpochSeconds(): number {
+  const raw = execFileSync('adb', ['shell', 'date', '+%s'], {
+    encoding: 'utf8',
+    timeout: 10_000,
+  });
+  const epoch = Number.parseInt(raw.trim(), 10);
+  if (!Number.isFinite(epoch) || epoch <= 0) {
+    throw new Error(`Failed to read device epoch time: ${raw.trim() || 'empty output'}`);
+  }
+  return epoch;
+}
+
 export function runReadinessCommand(): string {
   const method = process.env.PROBE_READINESS_METHOD ?? 'probeReadiness';
   const command = [
@@ -305,18 +348,37 @@ function summarizeReadinessError(raw: string): string {
 
 export function isProbeReadinessSufficient(
   readiness: ProbeReadiness,
-  minGraphChannels: number
+  minGraphChannels: number,
+  maxScoresSyncAgeS: number | null = null,
+  minScoresSyncTimestamp: number | null = null
 ): boolean {
   return (
     readiness.ready &&
     readiness.nodeRunning &&
     readiness.connectedPeers > 0 &&
     readiness.usableChannels > 0 &&
     readiness.syncHealthy &&
-    (readiness.graphChannelCount ?? 0) >= minGraphChannels
+    (readiness.graphChannelCount ?? 0) >= minGraphChannels &&
+    isScoresSyncFresh(readiness, maxScoresSyncAgeS, minScoresSyncTimestamp)
   );
 }
 
+function isScoresSyncFresh(
+  readiness: ProbeReadiness,
+  maxAgeS: number | null,
+  minTimestamp: number | null
+): boolean {
+  if (maxAgeS === null) return true;
+  const timestamp = readiness.latestPathfindingScoresSyncTimestamp;
+  if (!timestamp) return false;
+  if (Date.now() / 1000 - timestamp > maxAgeS) return false;
+  // Both timestamps come from the device clock; a post-reset sync happens
+  // seconds after the reset start, so strictly newer is the correct bound
+  // (a pre-reset sync can at most share the reset start second).
+  if (minTimestamp !== null && timestamp <= minTimestamp) return false;
+  return true;
+}
+
 export function summarizeProbeReadiness(readiness: ProbeReadiness): string {
   return [
     `running=${readiness.nodeRunning}`,
@@ -325,25 +387,35 @@ export function summarizeProbeReadiness(readiness: ProbeReadiness): string {
     `outboundSats=${readiness.outboundCapacitySats}`,
     `graphChannels=${readiness.graphChannelCount ?? 'n/a'}`,
     `graphNodes=${readiness.graphNodeCount ?? 'n/a'}`,
+    `scoresSync=${readiness.latestPathfindingScoresSyncTimestamp ?? 'n/a'}`,
     `syncHealthy=${readiness.syncHealthy}`,
     `ready=${readiness.ready}`,
   ].join(' ');
 }
 
 type WaitForProbeReadinessOptions = {
   logPrefix: string;
+  requireScoresSync?: boolean;
+  /** Device-clock epoch seconds; scores sync must be strictly newer than this (the reset start time). */
+  minScoresSyncTimestamp?: number | null;
 };
 
 export async function waitForProbeReadiness({
   logPrefix,
+  requireScoresSync = false,
+  minScoresSyncTimestamp = null,
 }: WaitForProbeReadinessOptions): Promise<ProbeReadiness> {
   const timeoutMs = parsePositiveIntEnv('PROBE_READINESS_TIMEOUT_MS') ?? DEFAULT_READINESS_TIMEOUT_MS;
   const pollMs = parsePositiveIntEnv('PROBE_READINESS_POLL_MS') ?? DEFAULT_READINESS_POLL_MS;
   const minGraphChannels =
     parseNonNegativeIntEnv('PROBE_MIN_GRAPH_CHANNELS') ?? DEFAULT_MIN_GRAPH_CHANNELS;
+  const maxScoresSyncAgeS = requireScoresSync
+    ? (parsePositiveIntEnv('PROBE_SCORES_SYNC_MAX_AGE_S') ?? DEFAULT_SCORES_SYNC_MAX_AGE_S)
+    : null;
+  const minSyncTimestamp = requireScoresSync ? minScoresSyncTimestamp : null;
 
   console.info(
-    `→ [${logPrefix}] Waiting for probe readiness (timeout ${timeoutMs / 1000}s, minGraphChannels ${minGraphChannels})...`
+    `→ [${logPrefix}] Waiting for probe readiness (timeout ${timeoutMs / 1000}s, minGraphChannels ${minGraphChannels}, requireScoresSync ${requireScoresSync})...`
   );
 
   const deadline = Date.now() + timeoutMs;
@@ -360,7 +432,7 @@ export async function waitForProbeReadiness({
     const readiness = raw ? parseProbeReadiness(raw) : null;
     if (readiness) {
       lastSummary = summarizeProbeReadiness(readiness);
-      if (isProbeReadinessSufficient(readiness, minGraphChannels)) {
+      if (isProbeReadinessSufficient(readiness, minGraphChannels, maxScoresSyncAgeS, minSyncTimestamp)) {
         console.info(`→ [${logPrefix}] Probe readiness satisfied: ${lastSummary}`);
         return readiness;
       }
@@ -409,6 +481,7 @@ export function renderProbeReport(
     '',
     `Required failures: ${failedRequired.length}`,
     `Probe order: ${probeOrderForReport()}`,
+    `Scores reset: ${scoresResetForReport()}`,
     `Readiness at probe start: ${readiness ? summarizeProbeReadiness(readiness) : 'not captured'}`,
     '',
     '| Target | Type | Amount sats | Required | Fetch | Probe | Retries | Duration ms | Failure |',
@@ -444,6 +517,14 @@ function probeOrderForReport(): string {
   }
 }
 
+function scoresResetForReport(): string {
+  try {
+    return String(resolveProbeResetScores());
+  } catch {
+    return `invalid (${process.env.PROBE_RESET_SCORES})`;
+  }
+}
+
 function parseProbeTarget(value: unknown): ProbeTarget {
   if (typeof value !== 'object' || value === null) {
     throw new Error('Each probe target must be an object');
@@ -560,6 +641,14 @@ export function parseNonNegativeIntEnv(name: string): number | null {
   return value;
 }
 
+function parseBooleanEnv(name: string): boolean | null {
+  const raw = process.env[name]?.trim().toLowerCase();
+  if (!raw) return null;
+  if (raw === 'true' || raw === '1' || raw === 'yes') return true;
+  if (raw === 'false' || raw === '0' || raw === 'no') return false;
+  throw new Error(`Invalid ${name} value: ${raw} (expected true or false)`);
+}
+
 function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }

test/specs/mainnet/probe.e2e.ts

@@ -6,7 +6,9 @@ import {
   parseNonNegativeIntEnv,
   parseProbeCommandSuccess,
   probeModeForTargetType,
+  resetPathfindingScores,
   resolveProbeOrder,
+  resolveProbeResetScores,
   resolveProbeTargets,
   runProbeInvoiceCommand,
   runProbeNodeCommand,
@@ -217,7 +219,17 @@ describe('@probe_mainnet - Lightning probe smoke', () => {
         expectAndroidAlert: false,
       });
       await waitForMainnetWalletReady({ logPrefix: 'Probe' });
-      readiness = await waitForProbeReadiness({ logPrefix: 'Probe' });
+
+      const resetScores = resolveProbeResetScores();
+      let scoresResetStartedAtS: number | null = null;
+      if (resetScores) {
+        scoresResetStartedAtS = await resetPathfindingScores({ logPrefix: 'Probe' });
+      }
+      readiness = await waitForProbeReadiness({
+        logPrefix: 'Probe',
+        requireScoresSync: resetScores,
+        minScoresSyncTimestamp: scoresResetStartedAtS,
+      });
 
       const probeOrder = resolveProbeOrder();
       const probes = buildProbeQueue(targets, probeOrder);