Skip to content

Commit 94fb5dc

Browse files
fix: force-kill worker pools and ptys on timeout/signal to prevent Windows CI hangs
On Windows, node-pty child processes survive process.exit() due to conpty handle inheritance (microsoft/node-pty#887). When the global timeout fires or the runner receives SIGTERM/SIGINT, worker pools were not terminated, leaving orphaned pty processes that cause GitHub Actions jobs to hang indefinitely. Changes: - Track active worker pools and terminate them before process.exit() - Add SIGTERM/SIGINT handlers in the runner for graceful shutdown - Track active terminals in workers and kill them on process exit Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
1 parent 18935f4 commit 94fb5dc

2 files changed

Lines changed: 45 additions & 5 deletions

File tree

src/runner/runner.ts

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,10 @@ type ExecutionOptions = {
4141

4242
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
4343

44+
const activePools = new Set<Pool>();
45+
4446
const createWorkerPool = (maxWorkers: number): Pool => {
45-
return workerpool.pool(path.join(__dirname, "worker.js"), {
47+
const pool = workerpool.pool(path.join(__dirname, "worker.js"), {
4648
workerType: "process",
4749
maxWorkers,
4850
forkOpts: {
@@ -54,6 +56,16 @@ const createWorkerPool = (maxWorkers: number): Pool => {
5456
},
5557
emitStdStreams: true,
5658
});
59+
activePools.add(pool);
60+
return pool;
61+
};
62+
63+
const terminateAllPools = async () => {
64+
const promises = [...activePools].map((pool) =>
65+
pool.terminate(true).catch(() => {})
66+
);
67+
await Promise.allSettled(promises);
68+
activePools.clear();
5769
};
5870

5971
const runSuites = async (
@@ -118,6 +130,7 @@ const runSuites = async (
118130
}
119131
await filePool.exec("afterAllWorker", []);
120132
} finally {
133+
activePools.delete(filePool);
121134
try {
122135
await filePool.terminate(true);
123136
} catch {
@@ -288,15 +301,28 @@ export const run = async (options: ExecutionOptions) => {
288301
}
289302
await reporter.start(allTests.length, shells, config.workers);
290303

304+
const forceExit = async (reason: string, code: number) => {
305+
console.error(reason);
306+
await terminateAllPools();
307+
process.exit(code);
308+
};
309+
291310
if (config.globalTimeout > 0) {
292311
setTimeout(() => {
293-
console.error(
294-
`Error: global timeout (${config.globalTimeout} ms) exceeded`
312+
void forceExit(
313+
`Error: global timeout (${config.globalTimeout} ms) exceeded`,
314+
1
295315
);
296-
process.exit(1);
297316
}, config.globalTimeout);
298317
}
299318

319+
process.on("SIGTERM", () => {
320+
void forceExit("Received SIGTERM, shutting down", 1);
321+
});
322+
process.on("SIGINT", () => {
323+
void forceExit("Received SIGINT, shutting down", 1);
324+
});
325+
300326
await runSuites(
301327
rootSuite.suites,
302328
new Set(allTests.map((test) => test.id)),

src/runner/worker.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { EventEmitter } from "node:events";
66
import workerpool from "workerpool";
77

88
import { Suite } from "../test/suite.js";
9-
import { spawn } from "../terminal/term.js";
9+
import { spawn, Terminal } from "../terminal/term.js";
1010
import { defaultShell } from "../terminal/shell.js";
1111
import { Snapshot, TestCase, TestStatus } from "../test/testcase.js";
1212
import { expect } from "../test/test.js";
@@ -15,6 +15,8 @@ import { poll } from "../utils/poll.js";
1515
import { flushSnapshotExecutionCache } from "../test/matchers/toMatchSnapshot.js";
1616
import { saveTrace, TracePoint } from "../trace/tracer.js";
1717

18+
const activeTerminals = new Set<Terminal>();
19+
1820
type WorkerResult = {
1921
error?: string;
2022
stdout?: string;
@@ -103,6 +105,7 @@ const runTest = async (
103105
trace,
104106
traceEmitter
105107
);
108+
activeTerminals.add(terminal);
106109

107110
// add slight delay for node-pty teardown
108111
let programExited: Promise<void> | undefined;
@@ -174,6 +177,7 @@ const runTest = async (
174177

175178
await Promise.resolve(test.testFunction(testArgs));
176179
} finally {
180+
activeTerminals.delete(terminal);
177181
try {
178182
terminal.kill();
179183
} catch {
@@ -456,6 +460,16 @@ if (!workerpool.isMainThread) {
456460
process.on("unhandledRejection", () => {
457461
// prevent worker crashes from unhandled promise rejections
458462
});
463+
process.on("exit", () => {
464+
for (const t of activeTerminals) {
465+
try {
466+
t.kill();
467+
} catch {
468+
// ignore
469+
}
470+
}
471+
activeTerminals.clear();
472+
});
459473
workerpool.worker({
460474
testWorker: testWorker,
461475
afterAllWorker: afterAllWorker,

0 commit comments

Comments
 (0)