fix(rotation): make selection token-bucket-aware; stop persisting local limiter

Fixes a cross-process regression introduced in 6.4.0. The 6.4.0 fix for
sticky/hybrid stalling on local token-bucket depletion wrote a synthetic
window into account.rateLimitResetTimes — but that field is persisted to the
shared accounts file and reloaded by every process, so one process draining
its OWN in-memory proactive limiter could spuriously rate-limit a
server-healthy account in OTHER concurrent processes (the multi-process /
PID-offset mode this tool targets).

Root-cause fix:
- TokenBucketTracker gains hasToken() / msUntilToken().
- AccountRotation selection (sticky, hybrid, round-robin) and
  getMinWaitTimeForFamily are now token-bucket-aware in-memory via a shared
  isSelectable() helper: a locally-depleted account is skipped with NO
  persisted state, and an all-depleted pool waits for refill (no spurious 503).
- The local-depletion branch in index.ts no longer calls recordRateLimit()
  (which applied a server-429-style health penalty to a purely-local throttle)
  and no longer writes a synthetic window. Removed the now-unused
  LOCAL_TOKEN_DEPLETION_COOLDOWN_MS constant.

Also fixes codex-warm reporting a quota-exhausted account as "warmed": a 429
is now classified by reason (quota/usage_limit -> exhausted/failed; transient
tokens/concurrent -> opened/warmed).

Adds cross-process-leak regression tests + warm 429-reason tests.
Full suite: 2613 passed, 1 skipped. Bumps to 6.4.1.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ndycode

.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "oc-codex-multi-auth",
-  "version": "6.4.0",
+  "version": "6.4.1",
   "description": "Install and operate oc-codex-multi-auth for OpenCode with ChatGPT Plus/Pro OAuth, Codex/GPT-5 routing, multi-account rotation, account switching, health checks, diagnostics, quota status, and recovery tools.",
   "skills": "./skills/",
   "interface": {

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "6.4.0"
+  ".": "6.4.1"
 }

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [6.4.1] - 2026-06-30
+
+### Fixed
+- Local token-bucket depletion no longer leaks into persisted, cross-process state. 6.4.0 made a depleted account rotate by writing a short synthetic window into `rateLimitResetTimes` — but that field is saved to the shared accounts file and reloaded by every process, so one process exhausting its own in-memory proactive limiter could spuriously mark a server-healthy account as rate-limited in OTHER concurrent processes (the multi-process/PID-offset deployment this tool targets). Account selection (`sticky`, `hybrid`, `round-robin`) and `getMinWaitTimeForFamily` are now token-bucket-aware directly: a locally-depleted account is skipped in-memory with no persisted state, and an all-depleted pool waits for token refill instead of returning a spurious 503. The local skip also no longer records a server-429-style health penalty, so a busy-but-healthy account is not deprioritized in `hybrid` scoring. (#183)
+- `codex-warm` no longer reports a quota-exhausted account as "warmed". A `429` is now classified by reason: a `quota`/`usage_limit` 429 (the window is already spent) is surfaced as a distinct failure, while a transient `tokens`/`concurrent` 429 (window active) still counts as warmed. (#182)
+
 ## [6.4.0] - 2026-06-30
 
 ### Added

index.ts

@@ -2139,33 +2139,28 @@ while (attempted.size < Math.max(1, accountCount)) {
 								// Consume a token before making the request for proactive rate limiting
 								const tokenConsumed = accountManager.consumeToken(account, modelFamily, model);
 								if (!tokenConsumed) {
-									accountManager.recordRateLimit(account, modelFamily, model);
-									// Mark a short, auto-expiring rate-limit window for this
-									// (family, model) so the depleted account becomes ineligible in
-									// `isRateLimitedForFamily`. Without this, drain-first `sticky`
-									// (and the `hybrid` fast-path) re-select the SAME depleted account
-									// on the next traversal iteration — `attempted` then trips the
-									// guard and the loop 503s while other accounts still have quota.
-									// The window also feeds `getMinWaitTimeForFamily`, so an all-
-									// depleted pool waits for refill instead of failing fast.
-									accountManager.markRateLimitedWithReason(
-										account,
-										ACCOUNT_LIMITS.LOCAL_TOKEN_DEPLETION_COOLDOWN_MS,
-										modelFamily,
-										"tokens",
-										model,
-									);
+									// Local (in-memory, per-process) proactive limiter is depleted for
+									// this account. The rotation selectors are token-bucket-aware, so
+									// they will not re-select this account until a token refills — no
+									// synthetic rate-limit window is written. Crucially we do NOT call
+									// recordRateLimit() here: that records a server-429-style health
+									// penalty and would mis-attribute a purely-local throttle as an
+									// upstream rejection. We also must not persist any local-limiter
+									// state (rateLimitResetTimes is written to the shared accounts file
+									// and would spuriously rate-limit healthy accounts in other
+									// processes). Just account the rotation and move on.
 									runtimeMetrics.accountRotations++;
 									runtimeMetrics.lastError =
 										`Local token bucket depleted for account ${account.index + 1} (${modelFamily}${model ? `:${model}` : ""})`;
 									runtimeMetrics.lastErrorCategory = "rate-limit-local";
 									logWarn(
 										`Skipping account ${account.index + 1}: local token bucket depleted for ${modelFamily}${model ? `:${model}` : ""}`,
 									);
-									// Skip THIS account and rotate to the next one. `account.index` is
-									// already in `attempted` (added above), so the traversal loop guard
-									// prevents reselecting it and terminates once all are exhausted.
-									// Using `break` here would abandon every other healthy account.
+									// Skip THIS account and rotate to the next one. The selector's
+									// token-bucket awareness guarantees it advances to an account with
+									// quota (or returns null so the wait/retry path engages via the
+									// token-refill wait in getMinWaitTimeForFamily). `break` would
+									// abandon every other healthy account.
 									continue;
 								}
 

lib/accounts/rotation.ts

@@ -28,6 +28,31 @@ import type { AccountState, ManagedAccount } from "./state.js";
 export class AccountRotation {
 	constructor(private readonly state: AccountState) {}
 
+	/**
+	 * Whether an account can serve a request for this (family, model) right now:
+	 * enabled, not rate-limited (real server 429s tracked in rateLimitResetTimes),
+	 * not cooling down, AND its in-memory local token bucket has a token.
+	 *
+	 * The token-bucket check is what keeps a locally-depleted account out of
+	 * selection. It is intentionally evaluated here (in-memory, per-process)
+	 * rather than by writing a synthetic window into the persisted
+	 * rateLimitResetTimes — that would leak a per-process proactive-limiter
+	 * signal into the cross-process accounts file and spuriously rate-limit
+	 * server-healthy accounts in other processes.
+	 */
+	private isSelectable(
+		account: ManagedAccount,
+		family: ModelFamily,
+		model?: string | null,
+	): boolean {
+		if (account.enabled === false) return false;
+		clearExpiredRateLimits(account);
+		if (isRateLimitedForFamily(account, family, model)) return false;
+		if (this.state.isAccountCoolingDown(account)) return false;
+		const quotaKey = model ? `${family}:${model}` : family;
+		return getTokenTracker().hasToken(account.index, quotaKey);
+	}
+
 	getCurrentOrNextForFamily(
 		family: ModelFamily,
 		model?: string | null,
@@ -41,15 +66,7 @@ export class AccountRotation {
 			const idx = (cursor + i) % count;
 			const account = this.state.accounts[idx];
 			if (!account) continue;
-			if (account.enabled === false) continue;
-
-			clearExpiredRateLimits(account);
-			if (
-				isRateLimitedForFamily(account, family, model) ||
-				this.state.isAccountCoolingDown(account)
-			) {
-				continue;
-			}
+			if (!this.isSelectable(account, family, model)) continue;
 
 			this.state.cursorByFamily[family] = (idx + 1) % count;
 			this.state.currentAccountIndexByFamily[family] = idx;
@@ -70,15 +87,7 @@ export class AccountRotation {
 			const idx = (cursor + i) % count;
 			const account = this.state.accounts[idx];
 			if (!account) continue;
-			if (account.enabled === false) continue;
-
-			clearExpiredRateLimits(account);
-			if (
-				isRateLimitedForFamily(account, family, model) ||
-				this.state.isAccountCoolingDown(account)
-			) {
-				continue;
-			}
+			if (!this.isSelectable(account, family, model)) continue;
 
 			this.state.cursorByFamily[family] = (idx + 1) % count;
 			account.lastUsed = nowMs();
@@ -102,15 +111,9 @@ export class AccountRotation {
 			if (currentAccount) {
 				if (currentAccount.enabled === false) {
 					// Fall through to hybrid selection.
-				} else {
-					clearExpiredRateLimits(currentAccount);
-					if (
-						!isRateLimitedForFamily(currentAccount, family, model) &&
-						!this.state.isAccountCoolingDown(currentAccount)
-					) {
-						currentAccount.lastUsed = nowMs();
-						return currentAccount;
-					}
+				} else if (this.isSelectable(currentAccount, family, model)) {
+					currentAccount.lastUsed = nowMs();
+					return currentAccount;
 				}
 			}
 		}
@@ -123,13 +126,9 @@ export class AccountRotation {
 			.map((account): AccountWithMetrics | null => {
 				if (!account) return null;
 				if (account.enabled === false) return null;
-				clearExpiredRateLimits(account);
-				const isAvailable =
-					!isRateLimitedForFamily(account, family, model) &&
-					!this.state.isAccountCoolingDown(account);
 				return {
 					index: account.index,
-					isAvailable,
+					isAvailable: this.isSelectable(account, family, model),
 					lastUsed: account.lastUsed,
 				};
 			})
@@ -177,20 +176,11 @@ export class AccountRotation {
 		const count = this.state.accounts.length;
 		if (count === 0) return null;
 
-		const isAvailable = (account: ManagedAccount): boolean => {
-			if (account.enabled === false) return false;
-			clearExpiredRateLimits(account);
-			return (
-				!isRateLimitedForFamily(account, family, model) &&
-				!this.state.isAccountCoolingDown(account)
-			);
-		};
-
 		// Prefer the account we are already pinned to while it still has quota.
 		const currentIndex = this.state.currentAccountIndexByFamily[family];
 		if (currentIndex >= 0 && currentIndex < count) {
 			const currentAccount = this.state.accounts[currentIndex];
-			if (currentAccount && isAvailable(currentAccount)) {
+			if (currentAccount && this.isSelectable(currentAccount, family, model)) {
 				currentAccount.lastUsed = nowMs();
 				return currentAccount;
 			}
@@ -201,7 +191,7 @@ export class AccountRotation {
 		for (let idx = 0; idx < count; idx++) {
 			const account = this.state.accounts[idx];
 			if (!account) continue;
-			if (!isAvailable(account)) continue;
+			if (!this.isSelectable(account, family, model)) continue;
 
 			this.state.currentAccountIndexByFamily[family] = idx;
 			this.state.cursorByFamily[family] = (idx + 1) % count;
@@ -326,19 +316,17 @@ export class AccountRotation {
 		const enabledAccounts = this.state.accounts.filter(
 			(account) => account.enabled !== false,
 		);
-		const available = enabledAccounts.filter((account) => {
-			clearExpiredRateLimits(account);
-			return (
-				!isRateLimitedForFamily(account, family, model) &&
-				!this.state.isAccountCoolingDown(account)
-			);
-		});
+		const available = enabledAccounts.filter((account) =>
+			this.isSelectable(account, family, model),
+		);
 		if (available.length > 0) return 0;
 		if (enabledAccounts.length === 0) return 0;
 
 		const waitTimes: number[] = [];
 		const baseKey = getQuotaKey(family);
 		const modelKey = model ? getQuotaKey(family, model) : null;
+		const tokenQuotaKey = model ? `${family}:${model}` : family;
+		const tokenTracker = getTokenTracker();
 
 		for (const account of enabledAccounts) {
 			const baseResetAt = account.rateLimitResetTimes[baseKey];
@@ -356,6 +344,16 @@ export class AccountRotation {
 			if (typeof account.coolingDownUntil === "number") {
 				waitTimes.push(Math.max(0, account.coolingDownUntil - now));
 			}
+
+			// An account blocked only by a depleted local token bucket becomes
+			// available again after refill; include that wait so a fully
+			// token-depleted pool waits for refill instead of returning 0 (503).
+			if (account.enabled !== false) {
+				const tokenWait = tokenTracker.msUntilToken(account.index, tokenQuotaKey);
+				if (tokenWait > 0 && Number.isFinite(tokenWait)) {
+					waitTimes.push(tokenWait);
+				}
+			}
 		}
 
 		return waitTimes.length > 0 ? Math.min(...waitTimes) : 0;

lib/accounts/warm-request.ts

@@ -17,6 +17,7 @@
 import { CODEX_BASE_URL } from "../constants.js";
 import { createCodexHeaders } from "../request/fetch-helpers.js";
 import { getCodexInstructions } from "../prompts/codex.js";
+import { parseRateLimitReason } from "./rate-limits.js";
 import type { RequestBody } from "../types.js";
 import { createLogger } from "../logger.js";
 
@@ -62,16 +63,33 @@ export async function buildWarmRequestBody(model = WARM_MODEL): Promise<RequestB
 	};
 }
 
+/**
+ * Outcome of a warm request.
+ * - `opened`: the request started/confirmed the usage window (2xx, or a 429
+ *   whose reason is a transient token/concurrency limit — the window is ticking).
+ * - `exhausted`: a 429 whose reason is quota/usage-limit — the account's window
+ *   is already spent, so warming it is meaningless. Reported distinctly so the
+ *   tool does not claim a quota-dead account was "warmed".
+ */
+export type WarmRequestStatus = "opened" | "exhausted";
+
+export interface WarmRequestResult {
+	status: WarmRequestStatus;
+	detail?: string;
+}
+
 /**
  * Send one warm request to open the account's usage window.
  *
- * Resolves `true` when the upstream accepted the request enough to start the
- * window. A 2xx clearly opens it; a 429 (rate-limited) means the window is
- * already open/active, which still satisfies the user intent ("the window is
- * ticking"). Any other non-2xx, or a network/timeout error, throws so the
- * caller records the account as failed.
+ * Resolves with `{ status: "opened" }` when the upstream started/confirmed the
+ * window (2xx, or a non-quota 429 meaning the window is already active), or
+ * `{ status: "exhausted" }` for a quota/usage-limit 429 (window already spent).
+ * Any other non-2xx, or a network/timeout error, throws so the caller records
+ * the account as failed.
  */
-export async function warmAccountWindow(params: WarmRequestParams): Promise<boolean> {
+export async function warmAccountWindow(
+	params: WarmRequestParams,
+): Promise<WarmRequestResult> {
 	const doFetch = params.fetchImpl ?? fetch;
 	const body = await buildWarmRequestBody();
 	const headers = createCodexHeaders(undefined, params.accountId, params.accessToken, {
@@ -92,27 +110,67 @@ export async function warmAccountWindow(params: WarmRequestParams): Promise<bool
 			signal: controller.signal,
 		});
 
-		// Drain/cancel the SSE stream immediately — we only needed to start the
-		// window, not consume the response.
+		if (response.ok) {
+			// Drain/cancel the SSE stream — we only needed to start the window.
+			try {
+				await response.body?.cancel();
+			} catch {
+				// Ignore cancellation failures.
+			}
+			return { status: "opened" };
+		}
+
+		// Read the error body (small) BEFORE classifying so a 429 quota-exhausted
+		// account is not mis-reported as warmed.
+		let bodyText = "";
 		try {
-			await response.body?.cancel();
+			bodyText = (await response.text()).slice(0, 2048);
 		} catch {
-			// Ignore cancellation failures.
+			// Ignore body-read failures; fall back to status-only classification.
 		}
 
-		if (response.ok) return true;
-
-		// 429 = window already open and currently rate-limited. The user's goal
-		// (window is ticking) is satisfied, so treat it as a successful warm.
 		if (response.status === 429) {
+			const reason = parseRateLimitReason(extractErrorCode(bodyText));
+			if (reason === "quota") {
+				log.debug("Warm ping hit 429 quota limit — account window already spent", {
+					accountId: params.accountId,
+				});
+				return { status: "exhausted", detail: "quota/usage limit reached" };
+			}
+			// token/concurrent/unknown → the window is active and ticking.
 			log.debug("Warm ping hit 429 — window already active", {
 				accountId: params.accountId,
+				reason,
 			});
-			return true;
+			return { status: "opened" };
 		}
 
 		throw new Error(`Warm request failed: HTTP ${response.status}`);
 	} finally {
 		clearTimeout(timeout);
 	}
 }
+
+/**
+ * Pull a rate-limit error code/message out of a JSON or plain-text error body
+ * so {@link parseRateLimitReason} can classify the 429. Best-effort: returns
+ * the raw text when JSON parsing fails so substring matching still works.
+ */
+function extractErrorCode(bodyText: string): string | undefined {
+	if (!bodyText) return undefined;
+	try {
+		const parsed = JSON.parse(bodyText) as {
+			error?: { code?: string; type?: string; message?: string };
+			code?: string;
+		};
+		return (
+			parsed.error?.code ??
+			parsed.error?.type ??
+			parsed.error?.message ??
+			parsed.code ??
+			bodyText
+		);
+	} catch {
+		return bodyText;
+	}
+}

lib/constants.ts

@@ -105,15 +105,4 @@ export const ACCOUNT_LIMITS = {
 	AUTH_FAILURE_COOLDOWN_MS: 30_000,
 	/** Number of consecutive auth failures before auto-removing account */
 	MAX_AUTH_FAILURES_BEFORE_REMOVAL: 3,
-	/**
-	 * Transient rate-limit window (ms) applied to an account when its local
-	 * client-side token bucket is depleted (proactive limiter, not a server
-	 * 429). Sized to ~one token refill at the default 6 tokens/min so the
-	 * account becomes selectable again as soon as a token regenerates. This
-	 * window makes the depleted account ineligible in `isRateLimitedForFamily`,
-	 * so every rotation strategy (including drain-first `sticky`, which would
-	 * otherwise re-pin the same depleted account and stall the request) advances
-	 * to the next account with quota instead of returning a spurious 503.
-	 */
-	LOCAL_TOKEN_DEPLETION_COOLDOWN_MS: 10_000,
 } as const;