fix(runtime): surface skip reason in pinned 503 (#486) (#487)

* fix(runtime): surface skip reason in pinned 503 (#486)

The pinned-account 503 response previously omitted the runtime skip
reason, forcing users to consult `codex-multi-auth status` out of band
and making remote diagnosis impossible. The response body now carries a
structured `reason` field and an `account_skip_reasons` map, mirroring
the existing `writePoolExhausted` shape, and the human-readable message
appends the same reason in parentheses.

A missing reason maps to explicit `null` and is captured in
`status.lastError` so a forecast vs. runtime state desync is detectable
instead of silently masked. Updates the error-contract reference doc to
match.

Adds end-to-end coverage for the rate-limited, cooling-down, and
disabled pinned-503 paths and extends the chooseAccount unit tests to
assert skipReasons map population for every pinned unavailability case
(rate-limited, cooling-down, disabled, policy-blocked, missing,
already-attempted).

Closes #486 partial: the diagnostic surface lands now; the underlying
state desync that prompted the report still needs logs from the
reporter to root-cause.

* fix(runtime): address review feedback on pinned 503 (#486)

- Extract `buildPinnedUnavailableErrorBody` helper so the null-reason
  state-desync branch can be unit-tested directly without standing up a
  full proxy. The helper is exported alongside a typed
  `PinnedUnavailableErrorBody` interface so external consumers can rely
  on a stable shape.
- Tighten the end-to-end cooling-down assertion from a regex prefix to
  an exact equality check against `cooling-down:auth-failure`, the
  string set by `markAccountCoolingDown` in the test setup. Prevents
  silent contract drift on the cooldown reason format.
- Add chooseAccount unit cases for the remaining pinned skip reasons
  flagged by review: `workspace-disabled` (all workspaces disabled) and
  `circuit-open` (failure threshold tripped via `recordFailure`). The
  suite now mirrors the full enumeration in
  `AccountManager.getAccountRuntimeSkipReason`.
- Add direct unit coverage for `buildPinnedUnavailableErrorBody` over
  four shapes: empty map yields `reason: null` with no parenthetical in
  the message, populated map yields the reason plus the parenthetical,
  null `pinnedIndex` resolves to `pinnedAccountIndex: null` without
  throwing, and the full `account_skip_reasons` map is mirrored even
  when the pinned index has no entry of its own.

Closes #486 partial: review feedback addressed; underlying state desync
still needs reporter logs to root-cause.

Author: ndycode

docs/reference/error-contracts.md

@@ -77,7 +77,7 @@ The default-on localhost Responses proxy returns JSON error payloads with a stab
 | `codex_pinned_account_unavailable` | `503` | A manual pin is set (via `codex-multi-auth switch`) but the pinned account is rate-limited, cooling down, disabled, or blocked by policy. Run `codex-multi-auth status` for details, or `codex-multi-auth unpin` to allow rotation |
 | `codex_runtime_rotation_proxy_error` | `500` | Proxy failed before forwarding the request |
 
-Pool exhaustion includes a `reason`, `retry_after_ms`, and a hint to run `codex-multi-auth rotation status`. Pinned-account-unavailable responses include a `pinnedAccountIndex` field identifying the pinned account.
+Pool exhaustion includes a `reason`, `retry_after_ms`, and a hint to run `codex-multi-auth rotation status`. Pinned-account-unavailable responses include a `pinnedAccountIndex` field identifying the pinned account, a structured `reason` field carrying the runtime skip reason (for example `rate-limited`, `cooling-down:auth-failure`, `circuit-open`, `disabled`, `workspace-disabled`, `policy-blocked`, `missing`, `already-attempted`) or `null` when no reason was recorded, and an `account_skip_reasons` map keyed by account index that mirrors the pool-exhausted response shape. The human-readable `message` appends the same reason in parentheses when present (see issue #486).
 
 ---
 

lib/runtime-rotation-proxy.ts

@@ -1044,6 +1044,47 @@ function writePoolExhausted(params: {
 	});
 }
 
+/**
+ * Build the JSON `error` body for a pinned-account 503 response. Extracted so
+ * the null-reason desync path (`reason: null`, no parenthetical in `message`)
+ * can be unit-tested without standing up a full proxy. The shape mirrors
+ * `writePoolExhausted` so consumers can handle both 503 codes uniformly. See
+ * issue #486.
+ */
+export interface PinnedUnavailableErrorBody {
+	message: string;
+	code: "codex_pinned_account_unavailable";
+	pinnedAccountIndex: number | null;
+	reason: string | null;
+	account_skip_reasons: Record<string, string>;
+}
+
+export function buildPinnedUnavailableErrorBody(
+	pinnedIndex: number | null | undefined,
+	accountSkipReasons: ReadonlyMap<number, string>,
+): PinnedUnavailableErrorBody {
+	const normalizedPinnedIndex =
+		typeof pinnedIndex === "number" ? pinnedIndex : null;
+	const skipReason =
+		normalizedPinnedIndex !== null
+			? accountSkipReasons.get(normalizedPinnedIndex) ?? null
+			: null;
+	const reasonSuffix = skipReason ? ` (${skipReason})` : "";
+	const displayIndex = (normalizedPinnedIndex ?? 0) + 1;
+	return {
+		message: `Pinned account ${displayIndex} is currently unavailable${reasonSuffix}; run \`codex-multi-auth status\` for details, or \`codex-multi-auth unpin\` to allow rotation.`,
+		code: "codex_pinned_account_unavailable",
+		pinnedAccountIndex: normalizedPinnedIndex,
+		reason: skipReason,
+		account_skip_reasons: Object.fromEntries(
+			[...accountSkipReasons.entries()].map(([index, reason]) => [
+				String(index),
+				reason,
+			]),
+		),
+	};
+}
+
 async function withTimeout<T>(
 	promise: Promise<T>,
 	timeoutMs: number,
@@ -1738,19 +1779,25 @@ export async function startRuntimeRotationProxy(
 			// When a manual pin is set and the pinned account is unavailable, do
 			// NOT silently fall through to rotation. Hard-fail with a 503 so the
 			// user is informed the pin cannot be honored. See issue #474.
+			//
+			// Surface the runtime skip reason in both the human-readable message
+			// and a structured `reason` field, mirroring `writePoolExhausted`. A
+			// null reason indicates a forecast/runtime state desync (the pinned
+			// account was selected but no skip reason was recorded) — see #486.
 			if (isPinned) {
+				const errorBody = buildPinnedUnavailableErrorBody(
+					pinnedIndex,
+					accountSkipReasons,
+				);
+				if (errorBody.reason === null) {
+					status.lastError = `pinned-503 missing skip reason (pinnedIndex=${pinnedIndex})`;
+				}
 				await usageRecorder?.record({
 					outcome: "failure",
 					statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
 					errorCode: "codex_pinned_account_unavailable",
 				});
-				writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
-					error: {
-						message: `Pinned account ${(pinnedIndex ?? 0) + 1} is currently unavailable; run \`codex-multi-auth status\` for details, or \`codex-multi-auth unpin\` to allow rotation.`,
-						code: "codex_pinned_account_unavailable",
-						pinnedAccountIndex: pinnedIndex,
-					},
-				});
+				writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, { error: errorBody });
 				return;
 			}
 

test/issue-474-pin-end-to-end.test.ts

@@ -279,8 +279,163 @@ describe("issue #474 — end-to-end pin honored over real HTTP proxy", () => {
 			expect(thirdResult.bodyText).toContain(
 				"codex_pinned_account_unavailable",
 			);
+			// Issue #486: the 503 body must surface the runtime skip reason so
+			// users can diagnose why the pin cannot be honored without scraping
+			// `codex-multi-auth status` logs out-of-band.
+			const thirdBody = JSON.parse(thirdResult.bodyText) as {
+				error: {
+					code: string;
+					pinnedAccountIndex: number | null;
+					reason: string | null;
+					account_skip_reasons: Record<string, string>;
+					message: string;
+				};
+			};
+			expect(thirdBody.error.reason).toBe("disabled");
+			expect(thirdBody.error.message).toContain("(disabled)");
+			expect(thirdBody.error.pinnedAccountIndex).toBe(otherAccountIndex);
+			expect(
+				thirdBody.error.account_skip_reasons[String(otherAccountIndex)],
+			).toBe("disabled");
 			// No additional upstream call — the proxy refused before issuing one.
 			expect(upstreamCalls).toHaveLength(2);
 		},
 	);
+
+	it(
+		"surfaces 'rate-limited' skip reason in pinned 503 body (issue #486)",
+		async () => {
+			const storagePath = makeTmpStoragePath();
+			const now = Date.now();
+			const initialStorage = createStorage(now);
+			const pinnedIndex = 1;
+			writeStorageFile(storagePath, {
+				...initialStorage,
+				pinnedAccountIndex: pinnedIndex,
+				affinityGeneration: 1,
+			});
+			setStoragePathDirect(storagePath);
+
+			const accountManager = new AccountManager(undefined, initialStorage);
+			openManagers.push(accountManager);
+
+			const pinned = accountManager.getAccountByIndex(pinnedIndex);
+			expect(pinned).not.toBeNull();
+			if (!pinned) throw new Error("setup failed");
+			// Match the family the proxy will resolve from `model: "gpt-5-codex"`.
+			// `getModelFamily("gpt-5-codex")` returns "gpt-5-codex", not "codex",
+			// so the rate-limit must be keyed under that family for the runtime
+			// skip-reason check to detect it.
+			accountManager.markRateLimitedWithReason(
+				pinned,
+				60_000,
+				"gpt-5-codex",
+				"quota",
+			);
+
+			const upstreamCalls: number[] = [];
+			const fetchImpl: typeof fetch = async (_input, init) => {
+				const headers = new Headers(init?.headers);
+				const auth = headers.get("authorization") ?? "";
+				const token = auth.replace(/^Bearer\s+/i, "");
+				const index = initialStorage.accounts.findIndex(
+					(a) => a.accessToken === token,
+				);
+				upstreamCalls.push(index);
+				return new Response(JSON.stringify({ ok: true, account: index }), {
+					status: HTTP_STATUS.OK,
+					headers: { "content-type": "application/json" },
+				});
+			};
+
+			const proxy = await startRuntimeRotationProxy({
+				accountManager,
+				fetchImpl,
+				upstreamBaseUrl: "https://example.test/backend-api",
+				clientApiKey: CLIENT_API_KEY,
+			});
+			openServers.push(proxy);
+
+			const result = await postViaHttp(
+				proxy,
+				{ model: "gpt-5-codex", stream: false },
+				"/v1/responses",
+			);
+			expect(result.status).toBe(HTTP_STATUS.SERVICE_UNAVAILABLE);
+			const body = JSON.parse(result.bodyText) as {
+				error: {
+					code: string;
+					reason: string | null;
+					account_skip_reasons: Record<string, string>;
+					message: string;
+				};
+			};
+			expect(body.error.code).toBe("codex_pinned_account_unavailable");
+			expect(body.error.reason).toBe("rate-limited");
+			expect(body.error.message).toContain("(rate-limited)");
+			expect(body.error.account_skip_reasons[String(pinnedIndex)]).toBe(
+				"rate-limited",
+			);
+			expect(upstreamCalls).toHaveLength(0);
+		},
+	);
+
+	it(
+		"surfaces a cooling-down skip reason in pinned 503 body (issue #486)",
+		async () => {
+			const storagePath = makeTmpStoragePath();
+			const now = Date.now();
+			const initialStorage = createStorage(now);
+			const pinnedIndex = 0;
+			writeStorageFile(storagePath, {
+				...initialStorage,
+				pinnedAccountIndex: pinnedIndex,
+				affinityGeneration: 1,
+			});
+			setStoragePathDirect(storagePath);
+
+			const accountManager = new AccountManager(undefined, initialStorage);
+			openManagers.push(accountManager);
+
+			const pinned = accountManager.getAccountByIndex(pinnedIndex);
+			if (!pinned) throw new Error("setup failed");
+			accountManager.markAccountCoolingDown(pinned, 60_000, "auth-failure");
+
+			const upstreamCalls: number[] = [];
+			const fetchImpl: typeof fetch = async () => {
+				upstreamCalls.push(-1);
+				return new Response("{}", { status: HTTP_STATUS.OK });
+			};
+
+			const proxy = await startRuntimeRotationProxy({
+				accountManager,
+				fetchImpl,
+				upstreamBaseUrl: "https://example.test/backend-api",
+				clientApiKey: CLIENT_API_KEY,
+			});
+			openServers.push(proxy);
+
+			const result = await postViaHttp(
+				proxy,
+				{ model: "gpt-5-codex", stream: false },
+				"/v1/responses",
+			);
+			expect(result.status).toBe(HTTP_STATUS.SERVICE_UNAVAILABLE);
+			const body = JSON.parse(result.bodyText) as {
+				error: {
+					code: string;
+					reason: string | null;
+					account_skip_reasons: Record<string, string>;
+					message: string;
+				};
+			};
+			expect(body.error.code).toBe("codex_pinned_account_unavailable");
+			expect(body.error.reason).toBe("cooling-down:auth-failure");
+			expect(body.error.message).toContain("(cooling-down:auth-failure)");
+			expect(body.error.account_skip_reasons[String(pinnedIndex)]).toBe(
+				"cooling-down:auth-failure",
+			);
+			expect(upstreamCalls).toHaveLength(0);
+		},
+	);
 });