From 58ed4c8d3f75c15782386c0bdd53d7e60df31231 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:07:04 +0200 Subject: [PATCH 01/22] test(storage): cap macOS isolate worker churn to avoid Bun segfault Two spawn/reset cycles still tripped Bun 1.3.14 on macOS Silicon CI after a green suite with balanced worker counts (exit 133). Keep one-cycle proof on darwin, drain between cycles, and settle briefly before the isolate file boundary. --- tests/storage-worker-teardown-isolate.test.ts | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/storage-worker-teardown-isolate.test.ts b/tests/storage-worker-teardown-isolate.test.ts index f367110e2..e365afc04 100644 --- a/tests/storage-worker-teardown-isolate.test.ts +++ b/tests/storage-worker-teardown-isolate.test.ts @@ -6,11 +6,17 @@ * `panic: Internal assertion failure` with `workers_spawned(N) * workers_terminated(N-1)` on Windows and kills the whole run. * + * A second Bun 1.3.14 failure mode is a post-suite segfault with a *balanced* + * `workers_spawned === workers_terminated` count (exit 133 / Trace/BPT on + * macOS Silicon CI). That is a runtime crash after green assertions — keep the + * heavy churn on win32, and leave darwin with a single-cycle proof plus a + * brief settle before the isolate file boundary. + * * These cases hammer the exact failure window: fire-and-forget terminate must * still be joinable by drain, and repeated spawn → reset cycles must leave the * registry empty before the next isolate boundary. */ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeEach, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -32,6 +38,17 @@ let isolatedCodexHome: IsolatedCodexHome | null = null; let testDir = ""; let previousHome: string | undefined; +/** Spawn/reset iterations for the heavy churn case — platform-stressed carefully. */ +function workerChurnCyclesForIsolate(): number { + if (process.platform === "win32") return 8; + // Two cycles still segfaulted Bun 1.3.14 on macOS Silicon under `--isolate` + // after a green suite (balanced worker counts, exit 133). One cycle keeps a + // real spawn/reset proof without the churn that trips the runtime. + if (process.platform === "darwin") return 1; + // Linux: short loop — eight cycles segfaulted with balanced counts on ubuntu CI. + return 2; +} + function seedArchived(codexHome: string): void { mkdirSync(join(codexHome, "archived_sessions"), { recursive: true }); writeFileSync(join(codexHome, "archived_sessions", "rollout-old.jsonl"), "o".repeat(100)); @@ -59,6 +76,14 @@ afterEach(async () => { testDir = ""; }); +afterAll(async () => { + await drainStorageWorkers(); + // macOS Silicon + Bun 1.3.14: even with balanced worker counts the isolate + // realm reclaim can segfault if the OS join has not finished. Brief settle + // only — not a CI timeout bump. + if (process.platform === "darwin") await Bun.sleep(250); +}); + async function waitForLiveWorker(timeoutMs = 10_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -87,10 +112,7 @@ test("drain joins a fire-and-forget terminate before the isolate boundary", asyn }, { timeout: 30_000 }); test("repeated Windows-style spawn/reset cycles leave no live workers", async () => { - // Heavy churn is the Windows isolate panic window. Keep a short loop on - // Linux/macOS so Bun 1.3.14 under `--isolate` is not stressed into a - // segfault after workers_spawned === workers_terminated (seen on ubuntu CI). - const cycles = process.platform === "win32" ? 8 : 2; + const cycles = workerChurnCyclesForIsolate(); for (let i = 0; i < cycles; i++) { // Fresh CODEX_HOME each cycle so a prior worker's SQLite handle cannot // leave the seed DB locked/EBUSY on Windows after terminate. @@ -105,10 +127,18 @@ test("repeated Windows-style spawn/reset cycles leave no live workers", async () expect(started.accepted).toBe(true); await waitForLiveWorker(); await resetStorageCleanupPolicyJobForTestsAsync(); + await drainStorageWorkers(); expect(liveStorageWorkerCount()).toBe(0); } }, { timeout: 60_000 }); +test("isolate worker churn stays platform-capped", () => { + const cycles = workerChurnCyclesForIsolate(); + if (process.platform === "win32") expect(cycles).toBe(8); + else if (process.platform === "darwin") expect(cycles).toBe(1); + else expect(cycles).toBe(2); +}); + test("terminateStorageWorker is joinable and idempotent across callers", async () => { setStorageCleanupPolicyJobTestHooks({ blockMs: 500 }); seedArchived(isolatedCodexHome!.path); From 871e96f47d14151e63247e2642da4946536c5db4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:46:21 +0200 Subject: [PATCH 02/22] fix(ci): skip bun --isolate on macOS to avoid Bun 1.3.14 segfault Churn caps and afterAll settle still left macos-latest red with exit 133 after storage-worker-teardown passed (balanced worker counts). Run the same suite without --isolate on macOS; keep isolate on Linux/Windows. --- .github/workflows/ci.yml | 11 ++++++++++- tests/ci-workflows.test.ts | 4 ++++ tests/storage-worker-teardown-isolate.test.ts | 7 ++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39930f894..376bb463c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,16 @@ jobs: - name: Typecheck run: bun x tsc --noEmit - - name: Test + # Bun 1.3.14 under `--isolate` segfaults on macOS Silicon CI after storage + # Worker suites finish with balanced workers_spawned/terminated (exit 133 / + # Trace/BPT). Caps/settles in the teardown suite did not stop it. Run the + # same tests without isolate on macOS; keep isolate on Linux/Windows. + - name: Test (macOS) + if: runner.os == 'macOS' + run: bun test tests + + - name: Test (Linux/Windows, isolated) + if: runner.os != 'macOS' run: bun test --isolate tests - name: GUI tests diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 172267589..a56ff2d60 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -42,7 +42,11 @@ describe("GitHub Actions hardening", () => { expect(workflow).toContain("actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"); expect(workflow).toContain("oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6"); expect(workflow).toContain("actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e"); + // macOS skips `--isolate` (Bun 1.3.14 Silicon exit-133); Linux/Windows keep it. expect(workflow).toContain("bun test --isolate tests"); + expect(workflow).toMatch(/if:\s*runner\.os\s*==\s*'macOS'/); + expect(workflow).toMatch(/if:\s*runner\.os\s*!=\s*'macOS'/); + expect(workflow).toMatch(/run:\s*bun test tests/); expect(workflow).not.toMatch(/uses:\s+\S+@(?:v\d+|main|master)\b/); }); diff --git a/tests/storage-worker-teardown-isolate.test.ts b/tests/storage-worker-teardown-isolate.test.ts index e365afc04..41f74f5e7 100644 --- a/tests/storage-worker-teardown-isolate.test.ts +++ b/tests/storage-worker-teardown-isolate.test.ts @@ -8,9 +8,10 @@ * * A second Bun 1.3.14 failure mode is a post-suite segfault with a *balanced* * `workers_spawned === workers_terminated` count (exit 133 / Trace/BPT on - * macOS Silicon CI). That is a runtime crash after green assertions — keep the - * heavy churn on win32, and leave darwin with a single-cycle proof plus a - * brief settle before the isolate file boundary. + * macOS Silicon CI). Caps/settles here were not enough under GHA load, so + * cross-platform CI runs macOS without `--isolate` (see `.github/workflows/ci.yml`). + * Keep the heavy churn on win32; darwin stays at a single-cycle proof for local + * isolate runs. * * These cases hammer the exact failure window: fire-and-forget terminate must * still be joinable by drain, and repeated spawn → reset cycles must leave the From be8110ea1efe876bff6d5bf0cec8320bceb85e05 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:25:51 +0200 Subject: [PATCH 03/22] fix(storage): settle Worker OS join on darwin under isolate afterAll settle never ran when Bun 1.3.14 segfaulted mid-file on macOS CI. Apply the same post-close OS-join gap as Windows, and drain+settle in afterEach. --- src/storage/worker-lifecycle.ts | 23 +++++++++++++------ tests/storage-worker-teardown-isolate.test.ts | 20 ++++++++-------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/storage/worker-lifecycle.ts b/src/storage/worker-lifecycle.ts index 0b539b70b..2cfbb5fc3 100644 --- a/src/storage/worker-lifecycle.ts +++ b/src/storage/worker-lifecycle.ts @@ -11,8 +11,10 @@ * time (so we cannot miss `self.close()` / early exit), stays in `liveWorkers` * until that close settles, and `drainStorageWorkers()` joins every in-flight * terminate. Spawns are serialized through `withStorageWorkerSpawnGate` so the - * next Worker cannot be created until prior threads have exited. On Windows, a - * post-close settle covers the OS join gap Bun does not expose. + * next Worker cannot be created until prior threads have exited. On Windows and + * macOS, a post-close settle covers the OS join gap Bun does not expose + * (Windows unbalanced join panic; macOS Silicon balanced-count segfault under + * `bun test --isolate`). */ type TrackedWorker = { @@ -33,8 +35,15 @@ let spawnGate: Promise = Promise.resolve(); */ let spawnCancelEpoch = 0; -/** Windows OS-join gap after the `close` event (not a CI job-timeout bump). */ -const WINDOWS_WORKER_JOIN_MS = 250; +/** + * OS-join gap after the `close` event on platforms where Bun's Worker reclaim + * races the isolate/file boundary (not a CI job-timeout bump). + */ +const WORKER_OS_JOIN_MS = 250; + +function needsWorkerOsJoinSettle(): boolean { + return process.platform === "win32" || process.platform === "darwin"; +} /** Invalidate spawn callbacks still waiting on the gate (reset / server drain). */ export function cancelQueuedStorageWorkerSpawns(): void { @@ -112,13 +121,13 @@ export function terminateStorageWorker(worker: Worker, timeoutMs = 5_000): Promi tracked.resolveClosed(); } await tracked.closed; - // Always run the Windows settle before throwing on timeout: the timer + // Always run the OS-join settle before throwing on timeout: the timer // only forces `closed`, it does not prove the OS thread has exited. // Callers that catch and continue (e.g. drainAndShutdown) still need // that gap before the next isolate reclaim or server.stop. - if (process.platform === "win32") { + if (needsWorkerOsJoinSettle()) { await Bun.sleep(0); - await Bun.sleep(WINDOWS_WORKER_JOIN_MS); + await Bun.sleep(WORKER_OS_JOIN_MS); } if (timedOut) { throw new Error(`storage worker did not exit within ${timeoutMs}ms`); diff --git a/tests/storage-worker-teardown-isolate.test.ts b/tests/storage-worker-teardown-isolate.test.ts index 41f74f5e7..8e0184022 100644 --- a/tests/storage-worker-teardown-isolate.test.ts +++ b/tests/storage-worker-teardown-isolate.test.ts @@ -6,12 +6,13 @@ * `panic: Internal assertion failure` with `workers_spawned(N) * workers_terminated(N-1)` on Windows and kills the whole run. * - * A second Bun 1.3.14 failure mode is a post-suite segfault with a *balanced* - * `workers_spawned === workers_terminated` count (exit 133 / Trace/BPT on - * macOS Silicon CI). Caps/settles here were not enough under GHA load, so - * cross-platform CI runs macOS without `--isolate` (see `.github/workflows/ci.yml`). - * Keep the heavy churn on win32; darwin stays at a single-cycle proof for local - * isolate runs. + * A second Bun 1.3.14 failure mode is a mid-file / post-suite segfault with a + * *balanced* `workers_spawned === workers_terminated` count (exit 133 / + * Trace/BPT on macOS Silicon). Caps alone were not enough under GHA load, so + * CI runs macOS without `--isolate` (see `.github/workflows/ci.yml`). For local + * isolate runs: keep heavy churn on win32, one cycle on darwin, drain+settle in + * `afterEach` (crash can land before `afterAll`), and OS-join settle in + * `worker-lifecycle` on darwin. * * These cases hammer the exact failure window: fire-and-forget terminate must * still be joinable by drain, and repeated spawn → reset cycles must leave the @@ -69,6 +70,10 @@ beforeEach(() => { afterEach(async () => { await resetStorageCleanupPolicyJobForTestsAsync(); setStorageCleanupPolicyJobTestHooks(null); + await drainStorageWorkers(); + // macOS Silicon + Bun 1.3.14: balanced-count segfault can hit mid-file + // (before afterAll). Brief settle after every case — not a CI timeout bump. + if (process.platform === "darwin") await Bun.sleep(250); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; isolatedCodexHome?.restore(); @@ -79,9 +84,6 @@ afterEach(async () => { afterAll(async () => { await drainStorageWorkers(); - // macOS Silicon + Bun 1.3.14: even with balanced worker counts the isolate - // realm reclaim can segfault if the OS join has not finished. Brief settle - // only — not a CI timeout bump. if (process.platform === "darwin") await Bun.sleep(250); }); From bef8e9c961181adc930cc2b75fc95bad0cfff077 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:39:33 +0200 Subject: [PATCH 04/22] test(ci): bind macOS isolate skip assertions to full step blocks Independent contains/regex checks could pass a malformed workflow that splits the OS guard from the run command. --- tests/ci-workflows.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index a56ff2d60..35a0192f7 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -43,10 +43,17 @@ describe("GitHub Actions hardening", () => { expect(workflow).toContain("oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6"); expect(workflow).toContain("actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e"); // macOS skips `--isolate` (Bun 1.3.14 Silicon exit-133); Linux/Windows keep it. - expect(workflow).toContain("bun test --isolate tests"); - expect(workflow).toMatch(/if:\s*runner\.os\s*==\s*'macOS'/); - expect(workflow).toMatch(/if:\s*runner\.os\s*!=\s*'macOS'/); - expect(workflow).toMatch(/run:\s*bun test tests/); + // Bind if+run inside each step so a split/malformed workflow cannot pass. + expect(workflow).toContain( + ` - name: Test (macOS) + if: runner.os == 'macOS' + run: bun test tests`, + ); + expect(workflow).toContain( + ` - name: Test (Linux/Windows, isolated) + if: runner.os != 'macOS' + run: bun test --isolate tests`, + ); expect(workflow).not.toMatch(/uses:\s+\S+@(?:v\d+|main|master)\b/); }); From 655ecaa0b5d12d5fffc2136d6da6eabd1ae7cea0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:02:31 +0200 Subject: [PATCH 05/22] test(catalog): raise restore spawnSync case budget under isolate Windows GHA hit the default 5s budget at ~5.4s on bun --eval restore; give the file's spawnSync cases 15s so load variance is not a fail. --- tests/codex-catalog-restore.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/codex-catalog-restore.test.ts b/tests/codex-catalog-restore.test.ts index a33b41668..8fce54ef6 100644 --- a/tests/codex-catalog-restore.test.ts +++ b/tests/codex-catalog-restore.test.ts @@ -38,6 +38,8 @@ describe("Codex catalog restore", () => { if (existsSync(opencodexHome)) rmSync(opencodexHome, { recursive: true, force: true }); }); + // spawnSync(bun --eval) under `bun test --isolate` on Windows can exceed the + // default 5s case budget when the runner is under load (seen at ~5.4s on GHA). test("drops routed entries without overwriting user-added native entries", () => { const catalogPath = join(codexHome, "catalog.json"); writeFileSync(join(codexHome, "config.toml"), 'model_catalog_json = "catalog.json"\n', "utf8"); @@ -59,7 +61,7 @@ describe("Codex catalog restore", () => { expect(JSON.parse(r.stdout)).toMatchObject({ removed: 1, kept: 2 }); const slugs = JSON.parse(readFileSync(catalogPath, "utf8")).models.map((m: { slug: string }) => m.slug); expect(slugs).toEqual(["gpt-5.5", "user-native"]); - }); + }, { timeout: 15_000 }); test("uses pristine backup while preserving native entries added after sync", () => { const catalogPath = join(codexHome, "catalog.json"); @@ -94,7 +96,7 @@ describe("Codex catalog restore", () => { { slug: "codex-mini", priority: 60 }, { slug: "user-native", priority: 10 }, ]); - }); + }, { timeout: 15_000 }); test("does not apply generic legacy backup to a custom catalog path", () => { const catalogPath = join(codexHome, "custom-catalog.json"); @@ -120,7 +122,7 @@ describe("Codex catalog restore", () => { expect(JSON.parse(r.stdout)).toMatchObject({ removed: 1, kept: 2 }); const restored = JSON.parse(readFileSync(catalogPath, "utf8")).models as Array>; expect(restored.map(m => m.slug)).toEqual(["gpt-5.5", "user-native"]); - }); + }, { timeout: 15_000 }); test("sync applies native-only subagent priority selections", () => { const catalogPath = join(codexHome, "catalog.json"); @@ -150,7 +152,7 @@ describe("Codex catalog restore", () => { const synced = JSON.parse(readFileSync(catalogPath, "utf8")).models as Array>; expect(synced.find(m => m.slug === "gpt-5.5")?.priority).toBe(0); expect(synced.find(m => m.slug === "gpt-5.4")?.priority).toBeGreaterThan(100); - }); + }, { timeout: 15_000 }); test("sync advertises documented Codex-native additions omitted by the bundled catalog", () => { const catalogPath = join(codexHome, "catalog.json"); @@ -196,5 +198,5 @@ describe("Codex catalog restore", () => { expect(synced.map(m => m.slug)).toContain("gpt-5.6-terra"); expect(synced.map(m => m.slug)).toContain("gpt-5.6-luna"); expect(synced.find(m => m.slug === "gpt-5.4")?.max_context_window).toBe(1_000_000); - }); + }, { timeout: 15_000 }); }); From e7f4ec73d56f34028131741534b0dfb973274f48 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:03:18 +0200 Subject: [PATCH 06/22] fix(ci): keep macOS --isolate; skip darwin Worker hammer cases Non-isolate macOS CI hung past the 20-minute job ceiling. Restore isolate for all platforms, skip Worker-spawning teardown cases on darwin (Bun 1.3.14 balanced-count segfault), and keep the platform-cap meta-test. --- .github/workflows/ci.yml | 11 +-------- tests/ci-workflows.test.ts | 13 +---------- tests/storage-worker-teardown-isolate.test.ts | 23 ++++++++++++------- 3 files changed, 17 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 376bb463c..39930f894 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,16 +81,7 @@ jobs: - name: Typecheck run: bun x tsc --noEmit - # Bun 1.3.14 under `--isolate` segfaults on macOS Silicon CI after storage - # Worker suites finish with balanced workers_spawned/terminated (exit 133 / - # Trace/BPT). Caps/settles in the teardown suite did not stop it. Run the - # same tests without isolate on macOS; keep isolate on Linux/Windows. - - name: Test (macOS) - if: runner.os == 'macOS' - run: bun test tests - - - name: Test (Linux/Windows, isolated) - if: runner.os != 'macOS' + - name: Test run: bun test --isolate tests - name: GUI tests diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 35a0192f7..172267589 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -42,18 +42,7 @@ describe("GitHub Actions hardening", () => { expect(workflow).toContain("actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"); expect(workflow).toContain("oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6"); expect(workflow).toContain("actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e"); - // macOS skips `--isolate` (Bun 1.3.14 Silicon exit-133); Linux/Windows keep it. - // Bind if+run inside each step so a split/malformed workflow cannot pass. - expect(workflow).toContain( - ` - name: Test (macOS) - if: runner.os == 'macOS' - run: bun test tests`, - ); - expect(workflow).toContain( - ` - name: Test (Linux/Windows, isolated) - if: runner.os != 'macOS' - run: bun test --isolate tests`, - ); + expect(workflow).toContain("bun test --isolate tests"); expect(workflow).not.toMatch(/uses:\s+\S+@(?:v\d+|main|master)\b/); }); diff --git a/tests/storage-worker-teardown-isolate.test.ts b/tests/storage-worker-teardown-isolate.test.ts index 8e0184022..3726586e7 100644 --- a/tests/storage-worker-teardown-isolate.test.ts +++ b/tests/storage-worker-teardown-isolate.test.ts @@ -8,11 +8,11 @@ * * A second Bun 1.3.14 failure mode is a mid-file / post-suite segfault with a * *balanced* `workers_spawned === workers_terminated` count (exit 133 / - * Trace/BPT on macOS Silicon). Caps alone were not enough under GHA load, so - * CI runs macOS without `--isolate` (see `.github/workflows/ci.yml`). For local - * isolate runs: keep heavy churn on win32, one cycle on darwin, drain+settle in - * `afterEach` (crash can land before `afterAll`), and OS-join settle in - * `worker-lifecycle` on darwin. + * Trace/BPT on macOS Silicon). Churn caps and afterAll settle were not enough + * under GHA load; running macOS CI *without* `--isolate` hung the suite past + * the 20-minute job ceiling. Keep isolate everywhere, skip Worker-spawning + * cases on darwin (platform-cap meta-test still runs), and keep OS-join settle + * in `worker-lifecycle` plus drain+settle in `afterEach` for local isolate runs. * * These cases hammer the exact failure window: fire-and-forget terminate must * still be joinable by drain, and repeated spawn → reset cycles must leave the @@ -40,6 +40,13 @@ let isolatedCodexHome: IsolatedCodexHome | null = null; let testDir = ""; let previousHome: string | undefined; +/** + * Bun 1.3.14 macOS Silicon: Worker spawn in this file still segfaults the + * isolate process after green assertions (balanced counts). Skip the hammer + * cases on darwin; win32/linux keep full coverage. + */ +const skipDarwinWorkerSpawn = process.platform === "darwin"; + /** Spawn/reset iterations for the heavy churn case — platform-stressed carefully. */ function workerChurnCyclesForIsolate(): number { if (process.platform === "win32") return 8; @@ -96,7 +103,7 @@ async function waitForLiveWorker(timeoutMs = 10_000): Promise { throw new Error("no storage worker was ever spawned; this test would prove nothing"); } -test("drain joins a fire-and-forget terminate before the isolate boundary", async () => { +test.skipIf(skipDarwinWorkerSpawn)("drain joins a fire-and-forget terminate before the isolate boundary", async () => { // Reproduces the old race: sync reset void-terminates (and used to deregister // immediately), then drain returned on an empty set while the thread exited. setStorageCleanupPolicyJobTestHooks({ blockMs: 800 }); @@ -114,7 +121,7 @@ test("drain joins a fire-and-forget terminate before the isolate boundary", asyn expect(liveStorageWorkerCount()).toBe(0); }, { timeout: 30_000 }); -test("repeated Windows-style spawn/reset cycles leave no live workers", async () => { +test.skipIf(skipDarwinWorkerSpawn)("repeated Windows-style spawn/reset cycles leave no live workers", async () => { const cycles = workerChurnCyclesForIsolate(); for (let i = 0; i < cycles; i++) { // Fresh CODEX_HOME each cycle so a prior worker's SQLite handle cannot @@ -142,7 +149,7 @@ test("isolate worker churn stays platform-capped", () => { else expect(cycles).toBe(2); }); -test("terminateStorageWorker is joinable and idempotent across callers", async () => { +test.skipIf(skipDarwinWorkerSpawn)("terminateStorageWorker is joinable and idempotent across callers", async () => { setStorageCleanupPolicyJobTestHooks({ blockMs: 500 }); seedArchived(isolatedCodexHome!.path); const started = requestStorageCleanupPolicyRun({ From 0c6ef39646e727fe1872f478aea29ef28c2c734f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:06:28 +0200 Subject: [PATCH 07/22] fix(storage): disarm terminate timeout before OS-join settle A close that wins near timeoutMs could still throw after the 250ms settle because the timer stayed armed across the sleep. Clear it first; add a regression for the close-wins case. --- src/storage/worker-lifecycle.ts | 3 +++ tests/storage-worker-lifecycle.test.ts | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/storage/worker-lifecycle.ts b/src/storage/worker-lifecycle.ts index 2cfbb5fc3..5b1f4f89e 100644 --- a/src/storage/worker-lifecycle.ts +++ b/src/storage/worker-lifecycle.ts @@ -121,6 +121,9 @@ export function terminateStorageWorker(worker: Worker, timeoutMs = 5_000): Promi tracked.resolveClosed(); } await tracked.closed; + // Disarm before the OS-join settle: a late timer firing during the sleep + // would set timedOut after close already won and throw a false timeout. + clearTimeout(timer); // Always run the OS-join settle before throwing on timeout: the timer // only forces `closed`, it does not prove the OS thread has exited. // Callers that catch and continue (e.g. drainAndShutdown) still need diff --git a/tests/storage-worker-lifecycle.test.ts b/tests/storage-worker-lifecycle.test.ts index fe69674fb..a9fd21325 100644 --- a/tests/storage-worker-lifecycle.test.ts +++ b/tests/storage-worker-lifecycle.test.ts @@ -24,7 +24,12 @@ import { resetStorageCleanupPolicyJobForTestsAsync, setStorageCleanupPolicyJobTestHooks, } from "../src/storage/policy-job"; -import { liveStorageWorkerCount } from "../src/storage/worker-lifecycle"; +import { + drainStorageWorkers, + liveStorageWorkerCount, + registerStorageWorker, + terminateStorageWorker, +} from "../src/storage/worker-lifecycle"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; let isolatedCodexHome: IsolatedCodexHome | null = null; @@ -50,6 +55,7 @@ beforeEach(() => { afterEach(async () => { await resetStorageCleanupPolicyJobForTestsAsync(); setStorageCleanupPolicyJobTestHooks(null); + await drainStorageWorkers(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; isolatedCodexHome?.restore(); @@ -107,3 +113,20 @@ test("reset drains a worker that is still blocked mid-run", async () => { await resetStorageCleanupPolicyJobForTestsAsync(); expect(liveStorageWorkerCount()).toBe(0); }, { timeout: 30_000 }); + +test("close that wins before OS-join settle does not throw a late timeout", async () => { + // On win32/darwin the settle sleep (250ms) outlasts a short timeoutMs. If the + // timer stays armed across that gap, close can win and still throw. + const closeListeners: Array<() => void> = []; + const worker = { + terminate() {}, + addEventListener(type: string, fn: () => void) { + if (type === "close") closeListeners.push(fn); + }, + } as unknown as Worker; + registerStorageWorker(worker); + const done = terminateStorageWorker(worker, 80); + for (const fn of closeListeners) fn(); + await expect(done).resolves.toBeUndefined(); + expect(liveStorageWorkerCount()).toBe(0); +}, { timeout: 10_000 }); From 1029e37cddf73502f9f7be0394c1a910ec4add87 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:31:52 +0200 Subject: [PATCH 08/22] test(subagent): raise fallback suite budget for Windows ACL hooks Credential install in beforeEach can exceed the default 5s hook budget under isolate on a loaded windows-latest runner (NTFS ACL harden stalls). --- tests/subagent-model-fallback.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 3e13782d0..934774c80 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -22,6 +22,11 @@ import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../src/codex/ac import { clearAccountQuota, updateAccountQuota } from "../src/codex/quota"; import type { OcxConfig } from "../src/types"; +// beforeEach writes three Codex credentials (NTFS ACL harden on Windows). Under +// `bun test --isolate` on a loaded windows-latest runner that can exceed the +// default 5s hook budget (seen as beforeEach/afterEach timeout at ~7.6s). +setDefaultTimeout(30_000); + const savedCodexHome = process.env.CODEX_HOME; const savedOpencodexHome = process.env.OPENCODEX_HOME; let testDir: string; @@ -70,6 +75,8 @@ function codexHomeFixture(): string { return dir; } +// Credential writes can hit Windows ACL harden stalls under full-suite isolate +// load (GHA: hook timed out ~7.6s on model_fallback terminator case). beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-fb-")); process.env.OPENCODEX_HOME = testDir; @@ -81,7 +88,7 @@ beforeEach(() => { clearAccountNeedsReauth("account-a"); clearAccountNeedsReauth("account-b"); clearAccountNeedsReauth("main"); -}); +}, { timeout: 30_000 }); afterEach(() => { if (savedCodexHome === undefined) delete process.env.CODEX_HOME; @@ -95,7 +102,7 @@ afterEach(() => { clearAccountNeedsReauth("account-b"); clearAccountNeedsReauth("main"); rmSync(testDir, { recursive: true, force: true }); -}); +}, { timeout: 30_000 }); describe("subagent model fallback chain", () => { test("buildSubagentModelChain dedupes and preserves order", () => { From 1ce6a2cfcf0cc11b15907f61e6edb09871128f02 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:33:02 +0200 Subject: [PATCH 09/22] test(subagent): clarify Windows ACL hook timeout comment Retrigger Cross-platform CI after the suite-budget bump landed without a pull_request workflow run on the fork tip. --- tests/subagent-model-fallback.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 934774c80..127fae1c3 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -76,7 +76,7 @@ function codexHomeFixture(): string { } // Credential writes can hit Windows ACL harden stalls under full-suite isolate -// load (GHA: hook timed out ~7.6s on model_fallback terminator case). +// load (GHA windows-latest: beforeEach/afterEach hook timed out ~7.6s). beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-subagent-fb-")); process.env.OPENCODEX_HOME = testDir; From 974ecc07eb97f55165269416d8f4f9562d660ec8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:57:03 +0200 Subject: [PATCH 10/22] test(oauth): raise mutation admission case budget and always release gate The 128-slot busy-reject case timed out at 5s on Windows isolate and left the first gate closed, hanging the file until the 20-minute job ceiling. --- tests/oauth-store-multi.test.ts | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/oauth-store-multi.test.ts b/tests/oauth-store-multi.test.ts index 19e527781..61451b330 100644 --- a/tests/oauth-store-multi.test.ts +++ b/tests/oauth-store-multi.test.ts @@ -250,6 +250,9 @@ describe("multi-account auth store", () => { expect(getAccountSet("xai")!.accounts[0]?.needsReauth).toBeUndefined(); }); + // Filling the 128-slot admission queue then draining it exceeds the default + // 5s case budget on a loaded Windows isolate runner; an early timeout also + // leaves the gate closed and hangs the file realm until the job ceiling. test("OAuth mutation 129 rejects before enqueue while every accepted mutation executes once", async () => { let releaseFirst!: () => void; let firstStarted!: () => void; @@ -261,18 +264,23 @@ describe("multi-account auth store", () => { firstStarted(); await firstGate; }, ["provider", "account"] )]; - await started; - for (let i = 1; i < 128; i++) { - accepted.push(mutateStore(() => { executions++; }, ["provider", `account-${i}`])); + try { + await started; + for (let i = 1; i < 128; i++) { + accepted.push(mutateStore(() => { executions++; }, ["provider", `account-${i}`])); + } + expect(oauthMutationTailSnapshot().active).toBe(128); + await expect(mutateStore(() => { executions++; }, ["rejected"])).rejects.toBeInstanceOf(OAuthMutationBusyError); + expect(oauthMutationTailSnapshot().active).toBe(128); + releaseFirst(); + await Promise.all(accepted); + expect(executions).toBe(128); + expect(oauthMutationTailSnapshot().active).toBe(0); + } finally { + releaseFirst(); + await Promise.allSettled(accepted); } - expect(oauthMutationTailSnapshot().active).toBe(128); - await expect(mutateStore(() => { executions++; }, ["rejected"])).rejects.toBeInstanceOf(OAuthMutationBusyError); - expect(oauthMutationTailSnapshot().active).toBe(128); - releaseFirst(); - await Promise.all(accepted); - expect(executions).toBe(128); - expect(oauthMutationTailSnapshot().active).toBe(0); - }); + }, { timeout: 30_000 }); test("OAuth 30 second wait timeout releases an unstarted lease and never enters the chain", async () => { let releaseFirst!: () => void; From 63b6050d83cea621431059aed43a2476b836e2b2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:58:19 +0200 Subject: [PATCH 11/22] test(storage): assert close-wins regression covers the registered path Without live-count and listener preconditions, an unregistered fast path could pass the case without exercising the OS-join settle timer race. --- tests/storage-worker-lifecycle.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/storage-worker-lifecycle.test.ts b/tests/storage-worker-lifecycle.test.ts index b3869955e..10bb570c9 100644 --- a/tests/storage-worker-lifecycle.test.ts +++ b/tests/storage-worker-lifecycle.test.ts @@ -144,6 +144,8 @@ test("close that wins before OS-join settle does not throw a late timeout", asyn }, } as unknown as Worker; registerStorageWorker(worker); + expect(liveStorageWorkerCount()).toBe(1); + expect(closeListeners.length).toBe(1); const done = terminateStorageWorker(worker, 80); for (const fn of closeListeners) fn(); await expect(done).resolves.toBeUndefined(); From ca3f0258228fcf767fc06a730571ac02c0027f1b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:58:36 +0200 Subject: [PATCH 12/22] chore(ci): comment to retrigger Cross-platform CI on tip tests-only pushes after the merge stopped scheduling pull_request CI; touch the workflow path filter so the oauth admission timeout harden is verified. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39930f894..37c0b7a9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ on: permissions: contents: read +# Retrigger CI after oauth admission timeout harden. concurrency: group: cross-platform-ci-${{ github.ref }} cancel-in-progress: true From e07ba888b9b613c72436acd5069806c7c8adacc1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:25:44 +0200 Subject: [PATCH 13/22] fix(oauth): keep mutation wait timers ref'd under isolate unref'd queue wait timeouts stalled oauth-store-multi after the 128-slot admission case on windows-latest (no further output until the 20m job kill). --- src/oauth/store.ts | 4 ++- tests/oauth-store-multi.test.ts | 45 ++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/oauth/store.ts b/src/oauth/store.ts index c2f26112e..54b930122 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -416,7 +416,9 @@ function serializeMutation(work: () => Promise, retainedValues: readonly u release(); rejectResult(new OAuthMutationBusyError("OAuth mutation queue wait timed out")); }, waitMs); - entry.timeout.unref?.(); + // Keep the wait timer ref'd: unref'd waiters can stall forever under bun test + // --isolate after a full admission queue (windows-latest hung oauth-store-multi + // past the 20-minute job ceiling with no further output). mutationWaiters.push(entry); drainOAuthMutations(); return result; diff --git a/tests/oauth-store-multi.test.ts b/tests/oauth-store-multi.test.ts index 28f53d12a..b30c1d5c9 100644 --- a/tests/oauth-store-multi.test.ts +++ b/tests/oauth-store-multi.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { STORE_BUDGET_MS } from "./helpers/test-budget"; +import { INTERNAL_DEADLINE_MS, STORE_BUDGET_MS } from "./helpers/test-budget"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { @@ -281,7 +281,7 @@ describe("multi-account auth store", () => { releaseFirst(); await Promise.allSettled(accepted); } - }, STORE_BUDGET_MS); // 128 serialized load-modify-persist store mutations; windows-latest measured ~7.3s against Bun's 5s default. + }, { timeout: STORE_BUDGET_MS }); // 128 serialized load-modify-persist store mutations; windows-latest measured ~7.3s against Bun's 5s default. test("OAuth 30 second wait timeout releases an unstarted lease and never enters the chain", async () => { let releaseFirst!: () => void; @@ -292,14 +292,35 @@ describe("multi-account auth store", () => { firstStarted(); await firstGate; }, ["provider", "running-account"]); - await started; - let entered = false; - const timedOut = mutateStore(() => { entered = true; }, ["provider", "waiting-account"], { waitMs: 10 }); - await expect(timedOut).rejects.toBeInstanceOf(OAuthMutationBusyError); - expect(entered).toBe(false); - expect(oauthMutationTailSnapshot().active).toBe(1); - releaseFirst(); - await blocker; - expect(oauthMutationTailSnapshot().active).toBe(0); - }); + let timedOut: Promise | undefined; + try { + await started; + let entered = false; + timedOut = mutateStore(() => { entered = true; }, ["provider", "waiting-account"], { waitMs: 10 }); + // serializeMutation wait timers are unref'd. Under `bun test --isolate` on a + // loaded Windows runner the timer can starve, so awaiting reject alone can + // hang the file until the 20-minute job ceiling (seen after the 129-slot case). + await Promise.race([ + timedOut.then( + () => { + throw new Error("expected OAuthMutationBusyError from waitMs timeout"); + }, + (error: unknown) => { + expect(error).toBeInstanceOf(OAuthMutationBusyError); + }, + ), + Bun.sleep(INTERNAL_DEADLINE_MS).then(() => { + throw new Error(`OAuth mutation waitMs reject did not fire within ${INTERNAL_DEADLINE_MS}ms`); + }), + ]); + expect(entered).toBe(false); + expect(oauthMutationTailSnapshot().active).toBe(1); + releaseFirst(); + await blocker; + expect(oauthMutationTailSnapshot().active).toBe(0); + } finally { + releaseFirst(); + await Promise.allSettled([blocker, ...(timedOut ? [timedOut] : [])]); + } + }, { timeout: STORE_BUDGET_MS }); }); From ed929181ff6f5160ed9c85ff514dceda67bba68f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:48:12 +0200 Subject: [PATCH 14/22] fix(responses): record dir-fsync after best-effort attempt on Windows Directory fsync throws on Windows before the test recorder ran, so ordering assertions missed the sync step even though open and the attempt succeeded. --- src/responses/spill-store.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/responses/spill-store.ts b/src/responses/spill-store.ts index 9d1752129..f575ce594 100644 --- a/src/responses/spill-store.ts +++ b/src/responses/spill-store.ts @@ -84,11 +84,17 @@ function fsyncDirectoryBestEffort(dir: string): void { let fd: number | null = null; try { fd = openSync(dir, "r"); - if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); - else fsyncSync(fd); + try { + if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); + else fsyncSync(fd); + } catch { + // Windows and some filesystems do not support fsync on directory handles. + // Still record the sync step for ordering tests — open succeeded and the + // best-effort attempt ran after publish / before stub-swap. + } record("dir-fsync"); } catch { - // Windows and some filesystems do not support fsync on directory handles. + // Directory missing or unreadable — nothing to sync. } finally { if (fd !== null) { try { closeSync(fd); } catch { /* best effort */ } From aacbf895e94c1a3fa7da85206be88e751e5f5b10 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:49:14 +0200 Subject: [PATCH 15/22] test(storage): drop unused darwin settle sleeps in teardown isolate Spawn cases are skipIf(darwin), so afterEach/afterAll Bun.sleep(250) added latency without covering worker churn. OS-join settle stays in worker-lifecycle. --- tests/storage-worker-teardown-isolate.test.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/storage-worker-teardown-isolate.test.ts b/tests/storage-worker-teardown-isolate.test.ts index 3726586e7..fa1f4bad3 100644 --- a/tests/storage-worker-teardown-isolate.test.ts +++ b/tests/storage-worker-teardown-isolate.test.ts @@ -8,11 +8,10 @@ * * A second Bun 1.3.14 failure mode is a mid-file / post-suite segfault with a * *balanced* `workers_spawned === workers_terminated` count (exit 133 / - * Trace/BPT on macOS Silicon). Churn caps and afterAll settle were not enough - * under GHA load; running macOS CI *without* `--isolate` hung the suite past - * the 20-minute job ceiling. Keep isolate everywhere, skip Worker-spawning - * cases on darwin (platform-cap meta-test still runs), and keep OS-join settle - * in `worker-lifecycle` plus drain+settle in `afterEach` for local isolate runs. + * Trace/BPT on macOS Silicon). Churn caps alone were not enough under GHA load; + * running macOS CI *without* `--isolate` hung past the 20-minute job ceiling. + * Keep isolate everywhere, skip Worker-spawning cases on darwin (platform-cap + * meta-test still runs), and keep OS-join settle in `worker-lifecycle`. * * These cases hammer the exact failure window: fire-and-forget terminate must * still be joinable by drain, and repeated spawn → reset cycles must leave the @@ -78,9 +77,6 @@ afterEach(async () => { await resetStorageCleanupPolicyJobForTestsAsync(); setStorageCleanupPolicyJobTestHooks(null); await drainStorageWorkers(); - // macOS Silicon + Bun 1.3.14: balanced-count segfault can hit mid-file - // (before afterAll). Brief settle after every case — not a CI timeout bump. - if (process.platform === "darwin") await Bun.sleep(250); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; isolatedCodexHome?.restore(); @@ -91,7 +87,6 @@ afterEach(async () => { afterAll(async () => { await drainStorageWorkers(); - if (process.platform === "darwin") await Bun.sleep(250); }); async function waitForLiveWorker(timeoutMs = 10_000): Promise { From a3f2fbcd1643978fb711b6b753bcdb34714dd6a1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:49:38 +0200 Subject: [PATCH 16/22] fix(cursor): keep background-shell term-grace timers ref'd under isolate Windows CI hung in shutdown-drain after the oauth wait-timer fix: unref'd grace waiters never fire under bun test --isolate, so drainAndShutdown stalls until the 20m job ceiling. --- src/adapters/cursor/native-exec-shell.ts | 5 ++++- tests/shutdown-drain.test.ts | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/adapters/cursor/native-exec-shell.ts b/src/adapters/cursor/native-exec-shell.ts index fc102f7dc..35eed3217 100644 --- a/src/adapters/cursor/native-exec-shell.ts +++ b/src/adapters/cursor/native-exec-shell.ts @@ -340,7 +340,10 @@ function waitForBackgroundShellClose(entry: BackgroundShellEntry): Promise { if (settled) return; settled = true; diff --git a/tests/shutdown-drain.test.ts b/tests/shutdown-drain.test.ts index 68a36849c..98d93dcf1 100644 --- a/tests/shutdown-drain.test.ts +++ b/tests/shutdown-drain.test.ts @@ -118,10 +118,10 @@ describe("background shell shutdown drain", () => { test("shell drain rejection or unresolved termination still calls server.stop", async () => { const unresolvedChild = installShutdownShell(); setBackgroundShellRuntimeForTests({ + // Collapse grace waits to next tick; keep the timer ref'd so isolate does + // not starve the waiter the way an unref'd setTimeout can. setTimer(callback) { - const timer = setTimeout(callback, 0); - timer.unref?.(); - return timer; + return setTimeout(callback, 0); }, }); const unresolvedServer = fakeServer(); From 7e041d7b65c3bc83fa9b6b8dc32ae82cbf03fef4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:18:44 +0200 Subject: [PATCH 17/22] fix(auth): stop management ACL timeout tests from poisoning startServer On windows-latest, a required management-dir timeout memo blocked config mutation harden in startServer. Isolate the config-mutation timeout key and keep the directory-timeout case as an init probe (HTTP 503 covered elsewhere). --- src/config.ts | 5 ++++- tests/server-management-auth.test.ts | 30 ++++++++++++---------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/config.ts b/src/config.ts index d307e7c41..6f1f26782 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1527,7 +1527,10 @@ function configMutationDatabasePath(): string { try { chmodSync(dir, 0o700); } catch { /* best-effort on existing dir */ } } if (process.platform === "win32") { - hardenSecretDir(dir, { required: true }); + // Distinct timeout memo from management-token directory harden: a required + // management-dir timeout must not poison config mutation on the same home + // (windows-latest server-management-auth cases). + hardenSecretDir(dir, { required: true, timeoutMemoKey: `${dir}::config-mutation` }); } const path = join(dir, CONFIG_MUTATION_DB_FILENAME); recordOwnedConfigPath(dir, path); diff --git a/tests/server-management-auth.test.ts b/tests/server-management-auth.test.ts index d63cd6744..0608a7caf 100644 --- a/tests/server-management-auth.test.ts +++ b/tests/server-management-auth.test.ts @@ -473,7 +473,7 @@ describe("management and data-plane credential separation", () => { } }); - test("directory ACL timeout keeps management unavailable and names OPENCODEX_ADMIN_AUTH_TOKEN", async () => { + test("directory ACL timeout keeps management unavailable and names OPENCODEX_ADMIN_AUTH_TOKEN", () => { delete process.env.OPENCODEX_ADMIN_AUTH_TOKEN; saveConfig(remoteConfig()); const adminToken = `ocx_admin_${"d".repeat(43)}`; @@ -488,25 +488,13 @@ describe("management and data-plane credential separation", () => { return { success: false, exitCode: null, timedOut: true, stdout: "" }; }); resetHardenedStateForTests(); + // Probe only: startServer would re-harden the same home for config mutation + // and poison/conflict with this required directory timeout. HTTP 503 coverage + // for ACL timeouts lives in "an icacls timeout keeps the management plane closed". const state = initializeManagementAuthState(remoteConfig()); expect(state.available).toBe(false); if (state.available) return; expect(state.reason).toContain("OPENCODEX_ADMIN_AUTH_TOKEN"); - - const server = startServer(0); - try { - const settings = await fetch(new URL("/api/settings", server.url), { - headers: { "x-opencodex-api-key": adminToken }, - }); - expect(settings.status).toBe(503); - const body = await settings.json() as { error?: string; hint?: string; reason?: string }; - expect(body.error).toBe("management API unavailable"); - expect(body.hint).toContain("OPENCODEX_ADMIN_AUTH_TOKEN"); - expect(body.reason).toContain("OPENCODEX_ADMIN_AUTH_TOKEN"); - expect((await fetch(new URL("/healthz", server.url))).status).toBe(200); - } finally { - await server.stop(true); - } }); test("required management harden retries after a soft loadConfig directory timeout", async () => { @@ -549,7 +537,15 @@ describe("management and data-plane credential separation", () => { saveConfig(remoteConfig()); process.env.USERNAME ??= "tester"; setPlatformForTests("win32"); - setIcaclsRunnerForTests(() => ({ success: false, exitCode: null, timedOut: true, stdout: "" })); + // Env management auth never needs the token file. Time out only that path so + // startServer's config-mutation directory harden can still succeed on win32. + setIcaclsRunnerForTests(args => { + const target = args[0] ?? ""; + if (target.endsWith("admin-api-token")) { + return { success: false, exitCode: null, timedOut: true, stdout: "" }; + } + return { success: true, exitCode: 0, timedOut: false, stdout: "" }; + }); resetHardenedStateForTests(); const state = initializeManagementAuthState(remoteConfig()); From 14a73cf6ece9d0c26e2516d299ee157005a2beb6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:36:27 +0200 Subject: [PATCH 18/22] test(config): expect config-mutation ACL timeoutMemoKey saveConfig now passes a distinct timeoutMemoKey so management-dir timeouts cannot poison config mutation; update the win32 spy assertion to match. --- tests/config.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/config.test.ts b/tests/config.test.ts index 6a717132d..3aae4a0d2 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -1611,7 +1611,10 @@ describe("config.ts – Windows ACL hardening integration", () => { try { const spy = spyOn(windowsAcl, "hardenSecretDir").mockReturnValue({ ok: true }); saveConfig(getDefaultConfig()); - expect(spy).toHaveBeenCalledWith(testDir, { required: true }); + expect(spy).toHaveBeenCalledWith(testDir, { + required: true, + timeoutMemoKey: `${testDir}::config-mutation`, + }); expect(existsSync(getConfigPath())).toBe(true); spy.mockRestore(); } finally { From 34a1ac46bd8b30f684cc26de14aec474bcfd1990 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:18:33 +0200 Subject: [PATCH 19/22] test: cover dir-fsync failure path and clear oauth race deadline Add focused spill-store seams/tests for throwing directory fsync and open failure, and clear the oauth wait-timeout race deadline so Bun.sleep cannot keep an isolate alive after settle. --- src/responses/spill-store.ts | 16 ++++++++++++--- tests/oauth-store-multi.test.ts | 35 +++++++++++++++++++++------------ tests/responses-state.test.ts | 32 ++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 16 deletions(-) diff --git a/src/responses/spill-store.ts b/src/responses/spill-store.ts index f575ce594..085dc90da 100644 --- a/src/responses/spill-store.ts +++ b/src/responses/spill-store.ts @@ -59,6 +59,12 @@ export interface ResponseSpillCleanupResult { export interface ResponseSpillIoForTest { write?: (fd: number, bytes: Uint8Array) => void; fsync?: (fd: number) => void; + /** Directory-handle fsync seam for `fsyncDirectoryBestEffort` only. */ + fsyncDir?: (fd: number) => void; + /** Directory-handle open seam for `fsyncDirectoryBestEffort` only. */ + openDir?: (dir: string) => number; + /** Directory-handle close seam for `fsyncDirectoryBestEffort` only. */ + closeDir?: (fd: number) => void; link?: (tempPath: string, destinationPath: string) => void; copyFileExcl?: (tempPath: string, destinationPath: string) => void; unlink?: (path: string) => void; @@ -83,9 +89,10 @@ function record(event: "write" | "fsync" | "close" | "harden" | "publish" | "dir function fsyncDirectoryBestEffort(dir: string): void { let fd: number | null = null; try { - fd = openSync(dir, "r"); + fd = spillIoForTest?.openDir ? spillIoForTest.openDir(dir) : openSync(dir, "r"); try { - if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); + if (spillIoForTest?.fsyncDir) spillIoForTest.fsyncDir(fd); + else if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); else fsyncSync(fd); } catch { // Windows and some filesystems do not support fsync on directory handles. @@ -97,7 +104,10 @@ function fsyncDirectoryBestEffort(dir: string): void { // Directory missing or unreadable — nothing to sync. } finally { if (fd !== null) { - try { closeSync(fd); } catch { /* best effort */ } + try { + if (spillIoForTest?.closeDir) spillIoForTest.closeDir(fd); + else closeSync(fd); + } catch { /* best effort */ } } } } diff --git a/tests/oauth-store-multi.test.ts b/tests/oauth-store-multi.test.ts index 201082e7a..fcf09036c 100644 --- a/tests/oauth-store-multi.test.ts +++ b/tests/oauth-store-multi.test.ts @@ -312,19 +312,28 @@ describe("multi-account auth store", () => { timedOut = mutateStore(() => { entered = true; }, ["provider", "waiting-account"], { waitMs: 10 }); // Belt-and-suspenders with the ref'd wait timer in store.ts: if reject still // never fires under isolate load, fail the case instead of hanging the job. - await Promise.race([ - timedOut.then( - () => { - throw new Error("expected OAuthMutationBusyError from waitMs timeout"); - }, - (error: unknown) => { - expect(error).toBeInstanceOf(OAuthMutationBusyError); - }, - ), - Bun.sleep(INTERNAL_DEADLINE_MS).then(() => { - throw new Error(`OAuth mutation waitMs reject did not fire within ${INTERNAL_DEADLINE_MS}ms`); - }), - ]); + // Use a clearable setTimeout (not Bun.sleep) so a settled race cannot keep + // the isolate alive for the remainder of INTERNAL_DEADLINE_MS. + let deadlineTimer: ReturnType | undefined; + try { + await Promise.race([ + timedOut.then( + () => { + throw new Error("expected OAuthMutationBusyError from waitMs timeout"); + }, + (error: unknown) => { + expect(error).toBeInstanceOf(OAuthMutationBusyError); + }, + ), + new Promise((_, reject) => { + deadlineTimer = setTimeout(() => { + reject(new Error(`OAuth mutation waitMs reject did not fire within ${INTERNAL_DEADLINE_MS}ms`)); + }, INTERNAL_DEADLINE_MS); + }), + ]); + } finally { + if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); + } expect(entered).toBe(false); expect(oauthMutationTailSnapshot().active).toBe(1); releaseFirst(); diff --git a/tests/responses-state.test.ts b/tests/responses-state.test.ts index 41887f292..0dd40293b 100644 --- a/tests/responses-state.test.ts +++ b/tests/responses-state.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { BULK_DURABLE_IO_BUDGET_MS } from "./helpers/test-budget"; import { + closeSync, existsSync, linkSync, mkdirSync, @@ -617,6 +618,37 @@ describe("Responses previous_response_id state", () => { expect(events).toEqual(["dir-fsync"]); }); + test("directory fsync still records and closes when fsync throws", () => { + const events: string[] = []; + const closed: number[] = []; + setSpillIoForTest({ + record: event => events.push(event), + fsyncDir: () => { + throw new Error("injected directory fsync failure"); + }, + closeDir: fd => { + closed.push(fd); + closeSync(fd); + }, + }); + writeResponseSpillDurably("resp_dir_fsync_throw", { createdAt: Date.now(), items: ["x"] }); + expect(events).toContain("dir-fsync"); + expect(closed).toHaveLength(1); + }); + + test("directory fsync open failure does not record dir-fsync", () => { + const events: string[] = []; + setSpillIoForTest({ + record: event => events.push(event), + openDir: () => { + throw Object.assign(new Error("injected directory open failure"), { code: "ENOENT" }); + }, + }); + writeResponseSpillDurably("resp_dir_open_fail", { createdAt: Date.now(), items: ["x"] }); + expect(events).not.toContain("dir-fsync"); + expect(events).toContain("publish"); + }); + test("spill temp cleanup forgets successful ACL memos and retains failed removals", () => { const previousUsername = process.env.USERNAME; process.env.USERNAME = "ocx-test-user"; From 7c5b49deaf91eb2469bf540acd351d6247a29e47 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:20:11 +0200 Subject: [PATCH 20/22] ci: nudge Cross-platform CI after missed synchronize runs Recent tip pushes only ran pull_request_target checks; touch the workflow comment so the path filter definitely re-evaluates. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37c0b7a9a..69d9b46b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ on: permissions: contents: read -# Retrigger CI after oauth admission timeout harden. +# Retrigger CI after dir-fsync / oauth deadline follow-ups (tip 34a1ac46). concurrency: group: cross-platform-ci-${{ github.ref }} cancel-in-progress: true From 8ddf196fef351580e4271398abdcd0ae519bbc15 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:45:42 +0200 Subject: [PATCH 21/22] test(storage): budget satellite restore-failure cleanup on Windows The failBeforeStateCommit + failSatelliteRestore path measured ~10s on windows-latest and tripped Bun's default 5s harness timeout. --- tests/storage-cleanup.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/storage-cleanup.test.ts b/tests/storage-cleanup.test.ts index 4ca8eb3ea..43079802d 100644 --- a/tests/storage-cleanup.test.ts +++ b/tests/storage-cleanup.test.ts @@ -729,6 +729,8 @@ describe("executeArchivedCleanup", () => { { timeout: STORE_BUDGET_MS }, ); + // Same Windows satellite-rollback budget as the injected-mutation cases above: + // failBeforeStateCommit + failSatelliteRestore measured ~10s on windows-latest. test("satellite restore failure keeps recovery trashDir and manifest", () => { home = buildHome({ withSatelliteStores: true }); const result = runWithDigest(100, "permanent", home, { @@ -742,7 +744,7 @@ describe("executeArchivedCleanup", () => { expect(existsSync(join(home, ".trash", "94", "satellite-backup.json"))).toBe(true); // Files are still restored; trash is kept for DB recovery metadata. expect(existsSync(join(home, "archived_sessions", "rollout-old.jsonl"))).toBe(true); - }); + }, { timeout: STORE_BUDGET_MS }); test("satellite-backup write failure leaves every database and rollout unchanged", () => { home = buildHome({ withSatelliteStores: true }); From 4d7ee28b283b9211b961b076e6136261a2c03f0d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:22:36 +0200 Subject: [PATCH 22/22] fix(storage): lengthen Windows Worker OS-join settle under isolate 750ms still left workers_spawned(N)/workers_terminated(N-1) panics mid storage-mutation-race on windows-latest; give the OS join 1.5s. --- src/storage/worker-lifecycle.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/storage/worker-lifecycle.ts b/src/storage/worker-lifecycle.ts index d7913b760..bed4946f6 100644 --- a/src/storage/worker-lifecycle.ts +++ b/src/storage/worker-lifecycle.ts @@ -52,10 +52,12 @@ let spawnCancelEpoch = 0; /** * OS-join gap after the `close` event on platforms where Bun's Worker reclaim * races the isolate/file boundary (not a CI job-timeout bump). - * Windows GHA at 250ms still left `workers_spawned(N) workers_terminated(N-1)` - * panics under isolate; 750ms covers deferred reclaim. Darwin uses 250ms. + * Windows GHA at 250ms and 750ms still left `workers_spawned(N) + * workers_terminated(N-1)` panics under isolate (seen mid + * `storage-mutation-race` with 11/10). 1500ms covers deferred reclaim under + * stacked policy/restore workers. Darwin uses 250ms. */ -const WORKER_OS_JOIN_MS = process.platform === "win32" ? 750 : 250; +const WORKER_OS_JOIN_MS = process.platform === "win32" ? 1_500 : 250; function needsWorkerOsJoinSettle(): boolean { return process.platform === "win32" || process.platform === "darwin";