Fix runtime rotation pool diagnostics (#480)

* fix: recover stale runtime rotation pool

* fix: serialize stale runtime reload recovery

* fix: address runtime diagnostics review bugs

* test: cover reset runtime review paths

Author: ndycode

lib/accounts.ts

@@ -17,6 +17,7 @@ import { MODEL_FAMILIES, type ModelFamily } from "./prompts/codex.js";
 import {
 	getHealthTracker,
 	getTokenTracker,
+	resetTrackers,
 	selectHybridAccount,
 	type AccountWithMetrics,
 	type HybridSelectionOptions,
@@ -39,7 +40,7 @@ import {
 	getAccountIdentityKey,
 	getRuntimeAccountIdentityKey,
 } from "./storage/identity.js";
-import { getCircuitBreaker } from "./circuit-breaker.js";
+import { getCircuitBreaker, resetAllCircuitBreakers } from "./circuit-breaker.js";
 import {
 	getStoragePathState,
 	runWithStoragePathState,
@@ -644,6 +645,31 @@ export class AccountManager {
 		);
 	}
 
+	getAccountRuntimeSkipReason(
+		index: number,
+		family: ModelFamily,
+		model?: string | null,
+	): string | null {
+		const account = this.getAccountByIndex(index);
+		if (!account) return "missing";
+		if (account.enabled === false) return "disabled";
+		if (!this.hasEnabledWorkspaces(account)) return "workspace-disabled";
+		clearExpiredRateLimits(account);
+		if (isRateLimitedForFamily(account, family, model)) return "rate-limited";
+		if (this.isAccountCoolingDown(account)) {
+			return account.cooldownReason
+				? `cooling-down:${account.cooldownReason}`
+				: "cooling-down";
+		}
+		if (!this.isCircuitAvailable(account)) return "circuit-open";
+		return null;
+	}
+
+	static resetVolatileRuntimeState(): void {
+		resetTrackers();
+		resetAllCircuitBreakers();
+	}
+
 	setActiveIndex(index: number): ManagedAccount | null {
 		if (!Number.isFinite(index)) return null;
 		if (index < 0 || index >= this.accounts.length) return null;

lib/codex-manager.ts

@@ -2542,6 +2542,7 @@ async function runForecast(args: string[]): Promise<number> {
 		formatWaitTime,
 		defaultDisplay: DEFAULT_DASHBOARD_DISPLAY_SETTINGS,
 		formatQuotaSnapshotLine,
+		loadRuntimeObservabilitySnapshot: loadPersistedRuntimeObservabilitySnapshot,
 	});
 }
 

lib/codex-manager/commands/forecast.ts

@@ -3,6 +3,7 @@ import { extractAccountEmail, sanitizeEmail } from "../../accounts.js";
 import {
 	buildForecastExplanation,
 	type ForecastAccountResult,
+	type RuntimeForecastOverlay,
 } from "../../forecast.js";
 import {
 	applyRefreshedAccountPatch,
@@ -22,6 +23,7 @@ interface ForecastCliOptions {
 	json: boolean;
 	explain: boolean;
 	model: string;
+	runtimeOverlay: boolean;
 }
 
 type ParsedArgsResult<T> =
@@ -80,6 +82,9 @@ export interface ForecastCommandDeps {
 			now: number;
 			refreshFailure?: TokenFailure;
 			liveQuota?: CodexQuotaSnapshot;
+			quotaCache?: QuotaCacheData | null;
+			allAccounts?: readonly AccountMetadataV3[];
+			runtimeOverlay?: RuntimeForecastOverlay | null;
 		}>,
 	) => ForecastAccountResult[];
 	summarizeForecast: (results: ForecastAccountResult[]) => {
@@ -107,6 +112,7 @@ export interface ForecastCommandDeps {
 	) => "success" | "warning" | "danger";
 	formatWaitTime: (ms: number) => string;
 	defaultDisplay: DashboardDisplaySettings;
+	loadRuntimeObservabilitySnapshot?: () => Promise<RuntimeForecastOverlay | null>;
 	logInfo?: (message: string) => void;
 	logError?: (message: string) => void;
 	getNow?: () => number;
@@ -131,6 +137,7 @@ function printForecastUsage(logInfo: (message: string) => void): void {
 			"  --json, -j         Print machine-readable JSON output",
 			"  --explain          Include structured recommendation reasoning",
 			"  --model, -m        Probe model for live mode (default: gpt-5.3-codex)",
+			"  --no-runtime-overlay  Ignore persisted runtime skip diagnostics",
 		].join("\n"),
 	);
 }
@@ -143,6 +150,7 @@ function parseForecastArgs(
 		json: false,
 		explain: false,
 		model: "gpt-5.3-codex",
+		runtimeOverlay: true,
 	};
 
 	for (let i = 0; i < args.length; i += 1) {
@@ -160,6 +168,10 @@ function parseForecastArgs(
 			options.explain = true;
 			continue;
 		}
+		if (arg === "--no-runtime-overlay") {
+			options.runtimeOverlay = false;
+			continue;
+		}
 		if (arg === "--model" || arg === "-m") {
 			const value = args[i + 1];
 			if (!value) return { ok: false, message: "Missing value for --model" };
@@ -205,7 +217,11 @@ export async function runForecastCommand(
 		? (await deps.loadDashboardDisplaySettings().catch(() => null)) ??
 			deps.defaultDisplay
 		: deps.defaultDisplay;
-	const quotaCache = options.live ? await deps.loadQuotaCache() : null;
+	const quotaCache = await deps.loadQuotaCache();
+	const runtimeOverlay =
+		options.runtimeOverlay && deps.loadRuntimeObservabilitySnapshot
+			? await deps.loadRuntimeObservabilitySnapshot().catch(() => null)
+			: null;
 	const workingQuotaCache = quotaCache
 		? deps.cloneQuotaCacheData(quotaCache)
 		: null;
@@ -346,6 +362,9 @@ export async function runForecastCommand(
 		now,
 		refreshFailure: refreshFailures.get(index),
 		liveQuota: liveQuotaByIndex.get(index),
+		quotaCache,
+		allAccounts: storage.accounts,
+		runtimeOverlay,
 	}));
 	const forecastResults = deps.evaluateForecastAccounts(forecastInputs);
 	const summary = deps.summarizeForecast(forecastResults);
@@ -373,6 +392,7 @@ export async function runForecastCommand(
 					command: "forecast",
 					model: requestedModel,
 					liveProbe: options.live,
+					runtimeOverlay: options.runtimeOverlay,
 					summary,
 					recommendation,
 					explanation: options.explain ? explanation : undefined,

lib/codex-manager/commands/report.ts

@@ -315,6 +315,15 @@ export async function runReportCommand(
 	const refreshFailures = new Map<number, TokenFailure>();
 	const liveQuotaByIndex = new Map<number, CodexQuotaSnapshot>();
 	const probeErrors: string[] = [];
+	let runtimeSnapshot: RuntimeObservabilitySnapshot | null | undefined;
+	let runtimeSnapshotLoadError: string | null = null;
+	try {
+		runtimeSnapshot = await deps.loadRuntimeObservabilitySnapshot?.();
+	} catch (error) {
+		runtimeSnapshot = null;
+		runtimeSnapshotLoadError = error instanceof Error ? error.message : String(error);
+		deps.logError?.(`Runtime observability snapshot unavailable: ${runtimeSnapshotLoadError}`);
+	}
 	let consideredLiveAccounts = 0;
 	let executedLiveProbes = 0;
 
@@ -453,6 +462,7 @@ export async function runReportCommand(
 					now,
 					refreshFailure: refreshFailures.get(index),
 					liveQuota: liveQuotaByIndex.get(index),
+					runtimeOverlay: runtimeSnapshot,
 				})),
 			)
 		: [];
@@ -521,7 +531,9 @@ export async function runReportCommand(
 				formatQuotaSnapshotLine,
 			),
 		},
-		runtime: await deps.loadRuntimeObservabilitySnapshot?.(),
+		runtimeOverlay: runtimeSnapshot !== null && runtimeSnapshot !== undefined,
+		runtime: runtimeSnapshot,
+		runtimeSnapshotLoadError,
 	};
 	if (report.forecast.recommendation.recommendedIndex !== null) {
 		const selectedIndex = report.forecast.recommendation.recommendedIndex;

lib/codex-manager/commands/rotation.ts

@@ -1,6 +1,11 @@
 import { existsSync, readFileSync, statSync } from "node:fs";
 import { join } from "node:path";
-import { formatAccountLabel, formatCooldown, formatWaitTime } from "../../accounts.js";
+import {
+	AccountManager,
+	formatAccountLabel,
+	formatCooldown,
+	formatWaitTime,
+} from "../../accounts.js";
 import { parseBooleanEnv } from "../../env-parsing.js";
 import { getCodexMultiAuthDir } from "../../runtime-paths.js";
 import { saveAccountsWithRetry } from "../forecast-report-shared.js";
@@ -32,7 +37,10 @@ import {
 	isQuotaCacheEntryExhausted,
 } from "../../quota-readiness.js";
 import type { QuotaCacheData } from "../../quota-cache.js";
-import type { RuntimeObservabilitySnapshot } from "../../runtime/runtime-observability.js";
+import {
+	recordRuntimeReset,
+	type RuntimeObservabilitySnapshot,
+} from "../../runtime/runtime-observability.js";
 import {
 	appRuntimeHelperStatusToSignal as appRuntimeHelperStatusToRuntimeSignal,
 	resolveAccountCurrentMarkers,
@@ -88,16 +96,84 @@ function printRotationUsage(logInfo: (message: string) => void): void {
 			"  codex-multi-auth rotation bind-app",
 			"  codex-multi-auth rotation unbind-app",
 			"  codex-multi-auth rotation reset-rate-limits [--all | --account <idx>] [--dry-run] [--json]",
+			"  codex-multi-auth rotation reset-runtime [--json]",
 			"",
 			"Behavior:",
 			"  - Runtime rotation is enabled by default for request-bearing Codex sessions",
 			"  - Binds the packaged Codex desktop app to the same localhost router when enabled or repaired",
 			"  - Use CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY=0 to disable the proxy for the current process without changing persistent settings",
 			"  - reset-rate-limits clears stored rateLimitResetTimes and active coolingDownUntil entries; use when `fix --live` confirms quota is available but the proxy still returns 503 pool-exhausted",
+			"  - reset-runtime clears process-local runtime trackers and re-applies the Codex app bind when available",
 		].join("\n"),
 	);
 }
 
+async function runResetRuntime(
+	args: string[],
+	deps: RotationCommandDeps,
+): Promise<number> {
+	const logInfo = deps.logInfo ?? console.log;
+	const logError = deps.logError ?? console.error;
+	let json = false;
+	for (const arg of args) {
+		if (arg === "--json" || arg === "-j") {
+			json = true;
+			continue;
+		}
+		if (arg === "--help" || arg === "-h" || arg === "help") {
+			logInfo("Usage: codex-multi-auth rotation reset-runtime [--json]");
+			return 0;
+		}
+		logError(`Unknown reset-runtime option: ${arg}`);
+		return 1;
+	}
+
+	AccountManager.resetVolatileRuntimeState();
+	recordRuntimeReset("rotation-reset-runtime");
+	let unbind: AppBindResult | null = null;
+	let bind: AppBindResult | null = null;
+	let appBindRestarted = false;
+	if (deps.unbindCodexApp && deps.bindCodexApp) {
+		try {
+			unbind = await deps.unbindCodexApp();
+			bind = await deps.bindCodexApp();
+			appBindRestarted = true;
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			if (json) {
+				logInfo(
+					JSON.stringify({
+						ok: false,
+						command: "rotation reset-runtime",
+						resetVolatileRuntimeState: true,
+						appBindRestarted,
+						error: message,
+					}),
+				);
+			} else {
+				logError(`Runtime reset completed, but app bind restart failed: ${message}`);
+			}
+			return 1;
+		}
+	}
+	const payload = {
+		ok: true,
+		command: "rotation reset-runtime",
+		resetVolatileRuntimeState: true,
+		appBindRestarted,
+		unbindStatus: unbind?.status.state ?? null,
+		bindStatus: bind?.status.state ?? null,
+	};
+	if (json) {
+		logInfo(JSON.stringify(payload));
+	} else {
+		logInfo("Runtime rotation volatile state reset.");
+		if (appBindRestarted) logInfo("Codex app bind restarted.");
+		else logInfo("Codex app bind helpers unavailable; new wrapper sessions will use the reset state.");
+	}
+	return 0;
+}
+
 interface ResetRateLimitsOptions {
 	scope: "all" | "account";
 	accountIndex: number | null;
@@ -660,6 +736,9 @@ export async function runRotationCommand(
 	if (subcommand === "reset-rate-limits") {
 		return runResetRateLimits(rest, deps);
 	}
+	if (subcommand === "reset-runtime") {
+		return runResetRuntime(rest, deps);
+	}
 	if (rest.length > 0) {
 		logError(`Unknown rotation option: ${rest[0]}`);
 		return 1;

lib/codex-manager/repair-commands.ts

@@ -37,6 +37,7 @@ import {
 import { setCodexCliActiveSelection } from "../codex-cli/writer.js";
 import { MODEL_FAMILIES, type ModelFamily } from "../prompts/codex.js";
 import { resolveNormalizedModel } from "../request/helpers/model-map.js";
+import { loadPersistedRuntimeObservabilitySnapshot } from "../runtime/runtime-observability.js";
 import type { AccountIdSource, TokenFailure, TokenResult } from "../types.js";
 
 type TokenSuccess = Extract<TokenResult, { type: "success" }>;
@@ -1922,6 +1923,9 @@ export async function runDoctor(
 		});
 
 		const now = Date.now();
+		const runtimeSnapshot =
+			(await loadPersistedRuntimeObservabilitySnapshot().catch(() => null)) ??
+			null;
 		const forecastResults = evaluateForecastAccounts(
 			storageForChecks.accounts.map((account, index) => ({
 				index,
@@ -1930,6 +1934,15 @@ export async function runDoctor(
 				now,
 			})),
 		);
+		const runtimeForecastResults = evaluateForecastAccounts(
+			storageForChecks.accounts.map((account, index) => ({
+				index,
+				account,
+				isCurrent: index === activeIndex,
+				now,
+				runtimeOverlay: runtimeSnapshot,
+			})),
+		);
 		const recommendation = recommendForecastAccount(forecastResults);
 		if (
 			recommendation.recommendedIndex !== null
@@ -1949,6 +1962,31 @@ export async function runDoctor(
 			});
 		}
 
+		const divergent = forecastResults.filter((result) => {
+			const runtimeResult = runtimeForecastResults[result.index];
+			return (
+				result.availability === "ready" &&
+				runtimeResult?.availability === "unavailable"
+			);
+		});
+		addCheck({
+			key: "forecast-runtime-alignment",
+			severity: divergent.length > 0 ? "warn" : "ok",
+			message:
+				divergent.length > 0
+					? `${divergent.length} account(s) look ready on disk but unavailable in runtime state`
+					: "Forecast and runtime availability are aligned",
+			details:
+				divergent.length > 0
+					? divergent
+							.map((result) => {
+								const runtimeResult = runtimeForecastResults[result.index];
+								return `account ${result.index + 1}: ${runtimeResult?.reasons.join("; ") || "runtime unavailable"}`;
+							})
+							.join(" | ")
+					: undefined,
+		});
+
 		if (activeExists) {
 			const activeAccount = storageForChecks.accounts[activeIndex];
 			const managerActiveEmail = sanitizeEmail(activeAccount?.email);