Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
4bdf451
fix(storage): await Worker join in test beforeEach under isolate
Wibias Aug 1, 2026
10ab2e8
fix(storage): join Workers via drainAndShutdown in isolate suites
Wibias Aug 1, 2026
4f45454
merge(dev): sync base for storage worker isolate fix
Wibias Aug 1, 2026
cc08831
fix(storage): clear mutation state only after Worker drain
Wibias Aug 1, 2026
65043ef
fix(server): await Bun Server.stop in drainAndShutdown
Wibias Aug 1, 2026
4ef29c8
fix(storage): cancel queued spawns on sync reset/abort
Wibias Aug 1, 2026
687533e
fix(gui): widen client-resource-poll waitFor on busy CI
Wibias Aug 1, 2026
05fc7e1
refactor(tests): drain storage workers before home allocation
Wibias Aug 1, 2026
b913fae
fix(storage): settle Worker join on darwin under isolate
Wibias Aug 1, 2026
ebe74ce
fix(ci): skip darwin Worker hammer cases under isolate
Wibias Aug 1, 2026
cd5afee
chore(ci): retrigger Cross-platform CI for tip
Wibias Aug 1, 2026
62994c1
merge(dev): resolve drainAndShutdown conflict for isolate fix
Wibias Aug 1, 2026
aeaf342
fix(gui): raise client-resource-poll test timeout above waitFor
Wibias Aug 1, 2026
85030c7
ci: raise Windows Cross-platform job timeout to 30m
Wibias Aug 1, 2026
30bd1f2
test(ci): expect 30-minute Cross-platform Windows job timeout
Wibias Aug 1, 2026
584d381
ci: raise Windows Cross-platform job timeout to 45m
Wibias Aug 1, 2026
60be0c5
fix(oauth): keep short mutation wait timers ref'd on Windows
Wibias Aug 1, 2026
b6c39ac
revert(ci): keep Windows Cross-platform job at 30m
Wibias Aug 1, 2026
56d1f8d
Merge remote-tracking branch 'upstream/dev' into fix/storage-worker-i…
Wibias Aug 1, 2026
f3bbc60
test(ci): pin Cross-platform timeout ownership per job
Wibias Aug 1, 2026
f27ed58
fix(test): skip isolate Worker hammers off Windows
Wibias Aug 1, 2026
2d82ddf
merge(dev): sync base for isolate hang fixes and Windows runner select
Wibias Aug 1, 2026
f1fa464
fix(test): narrow management-auth icacls stubs for real win32
Wibias Aug 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,13 @@ jobs:
# the result rather than the code under review — #711's rerun finished at
# 11.8min and passed while #653's was killed at 12.0min (issue #717).
# A cancelled job renders as `fail` in `gh pr checks`, so that flakiness
# reads as a broken PR. 20 minutes keeps a green Windows run green with
# real margin; it is not a licence for the suite to grow into it. If
# Windows approaches this ceiling too, fix the 2.5x platform gap instead
# of raising the number again.
timeout-minutes: 20
# reads as a broken PR. After the 2026-08-01 state-store admission merge,
# Windows under `bun test --isolate` on #827 hit the 20-minute kill while
# still green mid-suite (~19m of tests). 30 minutes is the margin for that
# tip — not a licence to absorb hung tests (see oauth mutation waitMs
# unref fix). Shrink the suite / close the platform gap rather than raising
# this again for ordinary variance.
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
Expand Down
19 changes: 14 additions & 5 deletions gui/tests/client-resource-poll.test.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import { afterEach, beforeEach, expect, test } from "bun:test";
import { afterEach, beforeEach, expect, test as bunTest } from "bun:test";
import { Window } from "happy-dom";
import { act, useEffect, useState } from "react";
import type { Root } from "react-dom/client";
import { useClientResource, useKeyedClientResource } from "../src/client-resource";

// Bun's default per-test budget is 5s. waitFor below is 15s (busy CI runners),
// so every case in this file needs a higher ceiling or the suite fails as
// "waitFor timed out" while the predicate never got its full window.
function test(name: string, fn: () => void | Promise<void>): void {
bunTest(name, fn, { timeout: 30_000 });
}

const globals = ["document", "window", "navigator", "IS_REACT_ACT_ENVIRONMENT"] as const;
let previousGlobals: Record<(typeof globals)[number], unknown>;
let testWindow: Window;
Expand All @@ -30,11 +37,13 @@ afterEach(() => {
* 1s was enough on an idle machine and not enough on a loaded CI runner: the
* visibility test waits for a poll that only fires after a real 20ms interval
* plus a React commit, and macOS CI blew through the budget mid-suite while the
* same file passed in isolation. The assertions below are about *whether* the
* fetch happens, never about how fast, so a longer ceiling costs nothing on a
* healthy run — it only stops a busy runner from reading as a product bug.
* same file passed in isolation. Windows GHA later timed out the same waits at
* 5s under suite load (`waitFor timed out` on #827). The assertions below are
* about *whether* the fetch happens, never about how fast, so a longer ceiling
* costs nothing on a healthy run — it only stops a busy runner from reading as
* a product bug.
*/
async function waitFor(predicate: () => boolean, timeoutMs = 5000): Promise<void> {
async function waitFor(predicate: () => boolean, timeoutMs = 15_000): Promise<void> {
const start = Date.now();
while (!predicate()) {
if (Date.now() - start > timeoutMs) throw new Error("waitFor timed out");
Expand Down
6 changes: 5 additions & 1 deletion src/oauth/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,11 @@ function serializeMutation<T>(work: () => Promise<T>, retainedValues: readonly u
release();
rejectResult(new OAuthMutationBusyError("OAuth mutation queue wait timed out"));
}, waitMs);
entry.timeout.unref?.();
// Only unref the long default wait. Short waitMs (tests) must stay ref'd:
// on Windows Bun under `bun test --isolate`, an unref'd timer can fail to
// fire while the head mutation holds an unresolved Promise, hanging the
// waiter forever (#827).
if (waitMs >= OAUTH_MUTATION_WAIT_MS) entry.timeout.unref?.();
mutationWaiters.push(entry);
drainOAuthMutations();
return result;
Expand Down
4 changes: 3 additions & 1 deletion src/server/lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,9 @@ export async function drainAndShutdown(
setStorageCleanupPolicyJobLiveApply(null);
} finally {
try {
s?.stop(true);
// Bun's Server.stop returns Promise<void>; fire-and-forget races the next
// isolate reclaim / follow-on listen the same way unterminated Workers did.
if (s) await s.stop(true);
} finally {
draining = false;
}
Expand Down
9 changes: 9 additions & 0 deletions src/storage/policy-job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,16 @@ function disownActiveRun(): void {
cancel?.();
}

/**
* Fire-and-forget reset. Prefer {@link resetStorageCleanupPolicyJobForTestsAsync}
* from test beforeEach/afterEach — sync terminate races Windows
* `bun test --isolate` reclaim when the next case spawns immediately.
*/
export function resetStorageCleanupPolicyJobForTests(): void {
disownActiveRun();
// Invalidate spawnGate callbacks that have not created a Worker yet — otherwise
// they can still spawn after this sync path releases the mutation slot.
cancelQueuedStorageWorkerSpawns();
if (activeWorker) {
void terminateStorageWorker(activeWorker);
activeWorker = null;
Expand Down Expand Up @@ -176,6 +184,7 @@ export async function resetStorageCleanupPolicyJobForTestsAsync(): Promise<void>
/** Terminate an in-flight worker during process shutdown. */
export function abortStorageCleanupPolicyJob(): void {
disownActiveRun();
cancelQueuedStorageWorkerSpawns();
if (activeWorker) {
void terminateStorageWorker(activeWorker);
activeWorker = null;
Expand Down
17 changes: 14 additions & 3 deletions src/storage/restore-job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,12 @@ export function setRestoreTrashJobTestHooks(hooks: RestoreJobTestHooks | null):
setStorageMutationCoordinatorTestHooks(hooks);
}

/**
* Fire-and-forget reset. Prefer {@link resetRestoreTrashJobForTestsAsync} from
* test beforeEach/afterEach under `bun test --isolate` on Windows.
*/
Comment thread
coderabbitai[bot] marked this conversation as resolved.
export function resetRestoreTrashJobForTests(): void {
cancelQueuedStorageWorkerSpawns();
if (activeWorker) {
void terminateStorageWorker(activeWorker);
activeWorker = null;
Expand All @@ -98,13 +103,19 @@ export async function resetRestoreTrashJobForTestsAsync(): Promise<void> {
cancelActiveRun?.();
cancelActiveRun = null;
testHooks = null;
resetStorageMutationCoordinatorForTests();
if (worker) await terminateStorageWorker(worker);
await drainStorageWorkers();
// Join the worker before clearing the mutation coordinator so a concurrent
// run cannot acquire CODEX_HOME while the aborted thread is still mutating.
try {
if (worker) await terminateStorageWorker(worker);
await drainStorageWorkers();
} finally {
resetStorageMutationCoordinatorForTests();
}
}

/** Terminate an in-flight worker during process shutdown. */
export function abortRestoreTrashJob(): void {
cancelQueuedStorageWorkerSpawns();
if (activeWorker) {
void terminateStorageWorker(activeWorker);
activeWorker = null;
Expand Down
23 changes: 18 additions & 5 deletions src/storage/worker-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,18 @@ let spawnGate: Promise<void> = Promise.resolve();
*/
let spawnCancelEpoch = 0;

/** Windows OS-join gap after the `close` event (not a CI job-timeout bump). */
const WINDOWS_WORKER_JOIN_MS = 250;
/**
* OS-join gap after the `close` event (not a CI job-timeout bump).
* Windows GHA at 250ms still left `workers_spawned(N) workers_terminated(N-1)`
* panics; 750ms covers deferred reclaim. Darwin under `bun test --isolate`
* also segfaults when the next spawn follows too closely after terminate even
* with balanced spawned/terminated counts (Bun 1.3.14), so apply a shorter
* settle there too. Linux keeps the close-event join only.
*/
const WORKER_JOIN_SETTLE_MS =
process.platform === "win32" ? 750
: process.platform === "darwin" ? 100
: 0;

/** Invalidate spawn callbacks still waiting on the gate (reset / server drain). */
export function cancelQueuedStorageWorkerSpawns(): void {
Expand Down Expand Up @@ -161,13 +171,16 @@ export function terminateStorageWorker(worker: Worker, timeoutMs = 5_000): Promi
tracked.resolveClosed();
}
await tracked.closed;
// Always run the Windows settle before throwing on timeout: the timer
// Disarm before the OS-join settle: a late timer firing during the sleep
// would set timedOut after close already won and throw a false timeout.
clearTimeout(timer);
// Always run the platform settle before throwing on timeout: the timer
// only forces `closed`, it does not prove the OS thread has exited.
// Callers that catch and continue (e.g. drainAndShutdown) still need
// that gap before the next isolate reclaim or server.stop.
if (process.platform === "win32") {
if (WORKER_JOIN_SETTLE_MS > 0) {
await Bun.sleep(0);
await Bun.sleep(WINDOWS_WORKER_JOIN_MS);
await Bun.sleep(WORKER_JOIN_SETTLE_MS);
}
if (timedOut) {
throw new Error(`storage worker did not exit within ${timeoutMs}ms`);
Expand Down
9 changes: 4 additions & 5 deletions tests/api-storage-policy-already-running.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
seedArchived,
setStorageCleanupPolicyJobTestHooks,
startServer,
stopStorageCleanupScheduler,
stopPolicyServer,
uninstallPolicyApiHarness,
waitForJobIdle,
type PolicyApiHarness,
Expand All @@ -18,8 +18,8 @@ import {

let harness: PolicyApiHarness;

beforeEach(() => {
harness = installPolicyApiHarness("ocx-api-storage-policy-busy");
beforeEach(async () => {
harness = await installPolicyApiHarness("ocx-api-storage-policy-busy");
});

afterEach(async () => {
Expand Down Expand Up @@ -60,8 +60,7 @@ test("POST run rejects when a job is already running", async () => {

await waitForJobIdle(server.url, firstBody.job.startedAt);
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
}, { timeout: 30_000 });
9 changes: 4 additions & 5 deletions tests/api-storage-policy-mutation-busy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
seedArchived,
setArchivedCleanupJobTestHooks,
startServer,
stopStorageCleanupScheduler,
stopPolicyServer,
uninstallPolicyApiHarness,
waitForJobIdle,
type PolicyApiHarness,
Expand All @@ -18,8 +18,8 @@ import {

let harness: PolicyApiHarness;

beforeEach(() => {
harness = installPolicyApiHarness("ocx-api-storage-policy-mut-busy");
beforeEach(async () => {
harness = await installPolicyApiHarness("ocx-api-storage-policy-mut-busy");
});

afterEach(async () => {
Expand Down Expand Up @@ -78,8 +78,7 @@ test("storage_mutation_busy clears inflight so a later policy run can start", as
expect(retryDone.job.lastOutcome?.skipped).toBeUndefined();
expect(retryDone.job.lastOutcome?.removed).toBe(1);
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
}, { timeout: 30_000 });
9 changes: 4 additions & 5 deletions tests/api-storage-policy-put-race.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
seedArchived,
setStorageCleanupPolicyJobTestHooks,
startServer,
stopStorageCleanupScheduler,
stopPolicyServer,
uninstallPolicyApiHarness,
waitForJobIdle,
type PolicyApiHarness,
Expand All @@ -18,8 +18,8 @@ import {

let harness: PolicyApiHarness;

beforeEach(() => {
harness = installPolicyApiHarness("ocx-api-storage-policy-put-race");
beforeEach(async () => {
harness = await installPolicyApiHarness("ocx-api-storage-policy-put-race");
});

afterEach(async () => {
Expand Down Expand Up @@ -109,8 +109,7 @@ test("blocked worker completion preserves concurrent policy PUT edits", async ()
expect(typeof body.lastRun?.at).toBe("number");
expect(typeof body.nextRun).toBe("number");
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
}, { timeout: 30_000 });
9 changes: 4 additions & 5 deletions tests/api-storage-policy-run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import {
installPolicyApiHarness,
seedArchived,
startServer,
stopStorageCleanupScheduler,
stopPolicyServer,
uninstallPolicyApiHarness,
waitForJobIdle,
type PolicyApiHarness,
Expand All @@ -17,8 +17,8 @@ import {

let harness: PolicyApiHarness;

beforeEach(() => {
harness = installPolicyApiHarness("ocx-api-storage-policy-run");
beforeEach(async () => {
harness = await installPolicyApiHarness("ocx-api-storage-policy-run");
});

afterEach(async () => {
Expand Down Expand Up @@ -68,8 +68,7 @@ test("POST run starts job promptly; skipped/success land on GET", async () => {
expect(ranDone.lastRun?.removed).toBe(1);
expect(JSON.stringify(ranDone)).not.toContain(harness.isolatedCodexHome.path.replaceAll("\\", "\\\\"));
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
}, { timeout: 30_000 });
15 changes: 6 additions & 9 deletions tests/api-storage-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ import {
fetch,
installPolicyApiHarness,
startServer,
stopStorageCleanupScheduler,
stopPolicyServer,
uninstallPolicyApiHarness,
type PolicyApiHarness,
resetStorageCleanupPolicyJobForTestsAsync,
} from "./helpers/storage-policy-api";

let harness: PolicyApiHarness;

beforeEach(() => {
harness = installPolicyApiHarness("ocx-api-storage-policy");
beforeEach(async () => {
harness = await installPolicyApiHarness("ocx-api-storage-policy");
});

afterEach(async () => {
Expand All @@ -36,8 +36,7 @@ describe("storage cleanup policy API", () => {
expect(body.trigger.archivedBytesOver).toBeGreaterThan(0);
expect(body.job.status).toBe("idle");
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
});
Expand Down Expand Up @@ -68,8 +67,7 @@ describe("storage cleanup policy API", () => {
expect(again.enabled).toBe(false);
expect(again.trigger.archivedBytesOver).toBe(1024);
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
});
Expand All @@ -89,8 +87,7 @@ describe("storage cleanup policy API", () => {
const body = await res.json();
expect(body.error).toContain("target");
} finally {
await server.stop(true);
stopStorageCleanupScheduler();
await stopPolicyServer(server);
await resetStorageCleanupPolicyJobForTestsAsync();
}
});
Expand Down
18 changes: 10 additions & 8 deletions tests/ci-workflows.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,17 @@ function count(text: string, fragment: string): number {
describe("GitHub Actions hardening", () => {
test("cross-platform CI keeps bounded jobs and immutable action references", async () => {
const workflow = await readText(".github/workflows/ci.yml");
const ci = Bun.YAML.parse(workflow) as {
jobs?: Record<string, { "timeout-minutes"?: number } | undefined>;
};

// The cross-platform `test` job sits at 20 minutes: a green Windows run measured
// 11.8 min against 4.6 on Linux, and the previous 12-minute ceiling left ~12s of
// margin, so runner variance rather than the code decided the verdict (#717).
// `npm-global-smoke` stays at 8; it finishes in 1-2 minutes. The windows-runner
// selector job is bounded at 2 minutes — it only picks a label.
expect(count(workflow, "timeout-minutes: 20")).toBe(1);
expect(count(workflow, "timeout-minutes: 8")).toBe(1);
expect(count(workflow, "timeout-minutes: 2\n")).toBe(1);
// Job-scoped: a global count still passes if values are swapped between jobs.
// Pin ownership explicitly. `test` is 30m for Windows isolate margin on #827
// after state-store admission — do not raise again; fix hung tests instead
// (unref'd oauth waitMs / shell kill-grace). Selector stays at 2; smoke at 8.
expect(ci.jobs?.["select-windows-runner"]?.["timeout-minutes"]).toBe(2);
expect(ci.jobs?.test?.["timeout-minutes"]).toBe(30);
expect(ci.jobs?.["npm-global-smoke"]?.["timeout-minutes"]).toBe(8);
// Every job must stay bounded — an unbounded job can hang a queue for hours.
expect(count(workflow, "timeout-minutes:")).toBe(3);
expect(workflow).toContain("actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0");
Expand Down
Loading
Loading