Skip to content

Commit 0e1c525

Browse files
ndycodendycode
committed
Merge branch 'pr-463' into merge/pr-462-463
2 parents e0bbc85 + 6a14962 commit 0e1c525

11 files changed

Lines changed: 413 additions & 27 deletions

‎lib/auto-update-checker.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -537,8 +537,13 @@ function taskkillProcessTree(pid: number): Promise<boolean> {
537537
timeout.unref?.();
538538
try {
539539
const killer = spawn(
540-
"taskkill",
541-
["/PID", String(pid), "/T", "/F"],
540+
"cmd.exe",
541+
[
542+
"/d",
543+
"/s",
544+
"/c",
545+
`taskkill /PID ${pid} /T /F`,
546+
],
542547
{
543548
stdio: "ignore",
544549
windowsHide: true,

‎lib/request/fetch-helpers.ts‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -328,12 +328,28 @@ export function isWorkspaceDisabledError(
328328
code: unknown,
329329
bodyText: string,
330330
): boolean {
331+
const normalizedCode = typeof code === "string" ? code.trim().toLowerCase() : "";
332+
333+
if (status === 402) {
334+
const normalizedTokens = normalizedCode
335+
.split(/[^a-z0-9_]+/i)
336+
.map((token) => token.trim())
337+
.filter((token) => token.length > 0);
338+
return (
339+
normalizedTokens.includes("deactivated_workspace") ||
340+
/\bdeactivated_workspace\b/i.test(bodyText)
341+
);
342+
}
343+
331344
if (status !== 403) {
332345
return false;
333346
}
334347

335-
const normalizedCode = typeof code === "string" ? code.trim().toLowerCase() : "";
336348
const haystack = `${normalizedCode} ${bodyText}`.toLowerCase();
349+
const normalizedTokens = normalizedCode
350+
.split(/[^a-z0-9_]+/i)
351+
.map((token) => token.trim())
352+
.filter((token) => token.length > 0);
337353

338354
const disabledPatterns = [
339355
/workspace.*(?:disabled|expired|deactivated|terminated)/i,
@@ -362,11 +378,6 @@ export function isWorkspaceDisabledError(
362378
return true;
363379
}
364380

365-
const normalizedTokens = normalizedCode
366-
.split(/[^a-z0-9_]+/i)
367-
.map((token) => token.trim())
368-
.filter((token) => token.length > 0);
369-
370381
return normalizedTokens.some((token) => workspaceErrorCodes.has(token));
371382
}
372383

‎lib/runtime-rotation-proxy.ts‎

Lines changed: 89 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {
3434
loadRuntimePolicyState,
3535
type RuntimePolicyDecision,
3636
} from "./policy/runtime-policy.js";
37+
import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
3738
import { SessionAffinityStore } from "./session-affinity.js";
3839
import type { OAuthAuthDetails, RequestBody, TokenResult } from "./types.js";
3940
import { isRecord } from "./utils.js";
@@ -91,6 +92,7 @@ type ExhaustionReason =
9192
| "network-error"
9293
| "auth-failure"
9394
| "budget"
95+
| "deactivated"
9496
| "no-account";
9597
type RuntimeProxyHttpError = Error & {
9698
statusCode: number;
@@ -637,6 +639,26 @@ async function readErrorBody(response: Response): Promise<string> {
637639
}
638640
}
639641

642+
function extractErrorCodeFromBody(bodyText: string): string | null {
643+
if (!bodyText.trim()) return null;
644+
try {
645+
const parsed = JSON.parse(bodyText) as unknown;
646+
if (!isRecord(parsed)) return null;
647+
const directCode = parsed.code;
648+
if (typeof directCode === "string" && directCode.trim()) {
649+
return directCode.trim();
650+
}
651+
const maybeError = parsed.error;
652+
if (!isRecord(maybeError)) return null;
653+
const nestedCode = maybeError.code;
654+
return typeof nestedCode === "string" && nestedCode.trim()
655+
? nestedCode.trim()
656+
: null;
657+
} catch {
658+
return null;
659+
}
660+
}
661+
640662
function getQuotaWindowWaitMs(headers: Headers, prefix: string, now: number): number {
641663
const resetAfterSeconds = Number.parseInt(
642664
headers.get(`${prefix}-reset-after-seconds`) ?? "",
@@ -1013,12 +1035,18 @@ export async function startRuntimeRotationProxy(
10131035
);
10141036
const attemptedIndexes = new Set<number>();
10151037
let exhaustionReason: ExhaustionReason = "no-account";
1016-
const accountAttemptLimit = Math.max(
1038+
const accountCount = accountManager.getAccountCount();
1039+
const transientAttemptLimit = Math.max(
10171040
1,
1018-
Math.min(accountManager.getAccountCount(), maxRuntimeAccountAttempts),
1041+
Math.min(accountCount, maxRuntimeAccountAttempts),
10191042
);
1043+
let transientAttempts = 0;
1044+
let transientExhaustionReason: ExhaustionReason | null = null;
10201045

1021-
while (attemptedIndexes.size < accountAttemptLimit) {
1046+
while (
1047+
attemptedIndexes.size < accountCount &&
1048+
transientAttempts < transientAttemptLimit
1049+
) {
10221050
const selected = chooseAccount({
10231051
accountManager,
10241052
sessionAffinityStore,
@@ -1049,6 +1077,8 @@ export async function startRuntimeRotationProxy(
10491077
accountManager.refundToken(selected, context.family, context.model);
10501078
exhaustionReason = "auth-failure";
10511079
if (!refreshed.retryable) continue;
1080+
transientAttempts += 1;
1081+
transientExhaustionReason = "auth-failure";
10521082
status.retries += 1;
10531083
status.rotations += 1;
10541084
continue;
@@ -1064,6 +1094,8 @@ export async function startRuntimeRotationProxy(
10641094
"auth-failure",
10651095
);
10661096
exhaustionReason = "auth-failure";
1097+
transientAttempts += 1;
1098+
transientExhaustionReason = "auth-failure";
10671099
status.retries += 1;
10681100
status.rotations += 1;
10691101
continue;
@@ -1108,6 +1140,8 @@ export async function startRuntimeRotationProxy(
11081140
);
11091141
accountManager.saveToDiskDebounced();
11101142
exhaustionReason = "network-error";
1143+
transientAttempts += 1;
1144+
transientExhaustionReason = "network-error";
11111145
status.retries += 1;
11121146
status.rotations += 1;
11131147
continue;
@@ -1131,11 +1165,52 @@ export async function startRuntimeRotationProxy(
11311165
);
11321166
accountManager.saveToDiskDebounced();
11331167
exhaustionReason = "rate-limit";
1168+
transientAttempts += 1;
1169+
transientExhaustionReason = "rate-limit";
11341170
status.retries += 1;
11351171
status.rotations += 1;
11361172
continue;
11371173
}
11381174

1175+
if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
1176+
const bodyText = await readErrorBody(upstream);
1177+
const errorCode = extractErrorCodeFromBody(bodyText);
1178+
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
1179+
const accountWasEnabled =
1180+
accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
1181+
false;
1182+
accountManager.refundToken(
1183+
refreshed.account,
1184+
context.family,
1185+
context.model,
1186+
);
1187+
if (accountWasEnabled) {
1188+
accountManager.recordFailure(
1189+
refreshed.account,
1190+
context.family,
1191+
context.model,
1192+
);
1193+
accountManager.setAccountEnabled(refreshed.account.index, false);
1194+
accountManager.saveToDiskDebounced();
1195+
}
1196+
sessionAffinityStore?.forgetSession(context.sessionKey);
1197+
exhaustionReason = "deactivated";
1198+
status.retries += 1;
1199+
status.rotations += 1;
1200+
continue;
1201+
}
1202+
1203+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
1204+
res.end(bodyText);
1205+
await usageRecorder.record({
1206+
outcome: "failure",
1207+
statusCode: upstream.status,
1208+
errorCode,
1209+
account: refreshed.account,
1210+
});
1211+
return;
1212+
}
1213+
11391214
if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
11401215
await readErrorBody(upstream);
11411216
accountManager.refundToken(refreshed.account, context.family, context.model);
@@ -1147,6 +1222,8 @@ export async function startRuntimeRotationProxy(
11471222
);
11481223
accountManager.saveToDiskDebounced();
11491224
exhaustionReason = "auth-failure";
1225+
transientAttempts += 1;
1226+
transientExhaustionReason = "auth-failure";
11501227
status.retries += 1;
11511228
status.rotations += 1;
11521229
continue;
@@ -1163,6 +1240,8 @@ export async function startRuntimeRotationProxy(
11631240
);
11641241
accountManager.saveToDiskDebounced();
11651242
exhaustionReason = "server-error";
1243+
transientAttempts += 1;
1244+
transientExhaustionReason = "server-error";
11661245
status.retries += 1;
11671246
status.rotations += 1;
11681247
continue;
@@ -1283,10 +1362,15 @@ export async function startRuntimeRotationProxy(
12831362
}
12841363

12851364
if (
1286-
attemptedIndexes.size >= accountAttemptLimit &&
1287-
accountAttemptLimit < accountManager.getAccountCount()
1365+
transientAttempts >= transientAttemptLimit &&
1366+
attemptedIndexes.size < accountCount
12881367
) {
12891368
exhaustionReason = "budget";
1369+
} else if (
1370+
exhaustionReason === "deactivated" &&
1371+
transientExhaustionReason
1372+
) {
1373+
exhaustionReason = transientExhaustionReason;
12901374
}
12911375

12921376
await usageRecorder?.record({

‎scripts/codex-multi-auth.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ function resolveCliVersion() {
2121
return "";
2222
}
2323

24+
/**
25+
* @param {string[]} args
26+
* @returns {string[]}
27+
*/
2428
function normalizeStandaloneArgs(args) {
2529
if (args[0] === "auth") return args;
2630
const firstArg = args[0] ?? "";

‎scripts/codex-routing.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ const AUTH_SUBCOMMANDS = new Set([
2626
"debug",
2727
]);
2828

29+
/**
30+
* @param {string[]} args
31+
* @returns {string[]}
32+
*/
2933
export function normalizeAuthAlias(args) {
3034
if (args.length >= 2 && args[0] === "multi" && args[1] === "auth") {
3135
return ["auth", ...args.slice(2)];
@@ -36,6 +40,10 @@ export function normalizeAuthAlias(args) {
3640
return args;
3741
}
3842

43+
/**
44+
* @param {string[]} args
45+
* @returns {boolean}
46+
*/
3947
export function shouldHandleMultiAuthAuth(args) {
4048
if (args[0] !== "auth") return false;
4149
if (args.length === 1) return true;

‎scripts/test-model-matrix.js‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,15 @@ function runQuiet(command, commandArgs) {
148148
}
149149
}
150150

151+
function runQuietWindowsTaskkill(pid) {
152+
runQuiet("cmd.exe", [
153+
"/d",
154+
"/s",
155+
"/c",
156+
`taskkill /F /T /PID ${pid}`,
157+
]);
158+
}
159+
151160
export function registerSpawnedCodex(pid) {
152161
if (!Number.isInteger(pid) || pid <= 0) return;
153162
spawnedCodexPids.add(pid);
@@ -228,7 +237,7 @@ function stopCodexServersInternal() {
228237
spawnedCodexPids.clear();
229238
for (const pid of tracked) {
230239
if (process.platform === "win32") {
231-
runQuiet("taskkill", ["/F", "/T", "/PID", String(pid)]);
240+
runQuietWindowsTaskkill(pid);
232241
continue;
233242
}
234243
runQuiet("kill", ["-9", String(pid)]);

‎test/auto-update-checker.test.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -889,8 +889,13 @@ describe("auto-update-checker", () => {
889889
await flushUpdateStartup();
890890
await vi.advanceTimersByTimeAsync(25);
891891
expect(childProcess.spawn).toHaveBeenLastCalledWith(
892-
"taskkill",
893-
["/PID", "1234", "/T", "/F"],
892+
"cmd.exe",
893+
[
894+
"/d",
895+
"/s",
896+
"/c",
897+
"taskkill /PID 1234 /T /F",
898+
],
894899
expect.objectContaining({ stdio: "ignore", windowsHide: true }),
895900
);
896901
killer.emit("exit", 0, null);

‎test/fetch-helpers.test.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,12 +688,21 @@ describe('isWorkspaceDisabledError', () => {
688688
expect(isWorkspaceDisabledError(403, 'error.usage_not_included', '')).toBe(false);
689689
});
690690

691-
it('returns false for non-403 status even with disabled message', () => {
691+
it('returns true for 402 deactivated_workspace signals', () => {
692+
expect(isWorkspaceDisabledError(402, 'deactivated_workspace', '')).toBe(true);
693+
expect(isWorkspaceDisabledError(402, 'error.deactivated_workspace', '')).toBe(true);
694+
expect(
695+
isWorkspaceDisabledError(402, '', '{"error":{"code":"deactivated_workspace"}}'),
696+
).toBe(true);
697+
});
698+
699+
it('returns false for statuses without supported disabled signals', () => {
692700
expect(isWorkspaceDisabledError(400, '', 'Your workspace has been disabled')).toBe(false);
693701
expect(isWorkspaceDisabledError(401, '', 'Your workspace has been disabled')).toBe(false);
694702
expect(isWorkspaceDisabledError(500, '', 'Your workspace has been disabled')).toBe(false);
695703
expect(isWorkspaceDisabledError(400, 'workspace_disabled', '')).toBe(false);
696704
expect(isWorkspaceDisabledError(402, 'payment_required', '')).toBe(false);
705+
expect(isWorkspaceDisabledError(402, '', 'Your workspace has been disabled')).toBe(false);
697706
});
698707

699708
it('returns false for 403 with unrelated messages', () => {

‎test/index-retry.test.ts‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -694,7 +694,6 @@ describe("OpenAIAuthPlugin rate-limit retry", () => {
694694
});
695695

696696
it("keeps the total request cap when empty-response retries and server-error rotation combine", async () => {
697-
vi.useFakeTimers();
698697
const logger = await import("../lib/logger.js");
699698
const logWarnSpy = vi.spyOn(logger, "logWarn").mockImplementation(() => {});
700699

@@ -752,11 +751,7 @@ describe("OpenAIAuthPlugin rate-limit retry", () => {
752751
});
753752

754753
const sdk = (await plugin.auth.loader(getAuth, { options: {}, models: {} })) as any;
755-
const fetchPromise = sdk.fetch("https://example.com", {});
756-
757-
await vi.advanceTimersByTimeAsync(1500);
758-
759-
const response = await fetchPromise;
754+
const response = await sdk.fetch("https://example.com", {});
760755
const payload = await response.json();
761756

762757
expect(fetchMock).toHaveBeenCalledTimes(6);

0 commit comments

Comments
 (0)