Skip to content

Commit 8c9ee98

Browse files
xuiocodex
andcommitted
Harden MCP shutdown against orphan spins
Co-Authored-By: OpenAI Codex <noreply@openai.com>
1 parent 7d76736 commit 8c9ee98

9 files changed

Lines changed: 602 additions & 74 deletions

File tree

dist/index.js

Lines changed: 186 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21433,6 +21433,7 @@ var logWriter = (line) => {
2143321433
writeDefaultLog(line);
2143421434
};
2143521435
var lastLogFileError;
21436+
var stderrMirrorDisabled = false;
2143621437
function configuredLogProfile(env = process.env) {
2143721438
const raw = env.CODEX_SUBAGENTS_LOG_PROFILE?.trim().toLowerCase();
2143821439
return raw === "production" ? "production" : "debug";
@@ -21535,6 +21536,9 @@ var logger = {
2153521536
rawWarn: (event, fields) => logRaw("warn", event, fields),
2153621537
rawError: (event, fields) => logRaw("error", event, fields)
2153721538
};
21539+
function disableStderrLogMirrorForShutdown() {
21540+
stderrMirrorDisabled = true;
21541+
}
2153821542
function loggingDiagnostics(env = process.env) {
2153921543
const logFile = env.CODEX_SUBAGENTS_LOG_FILE?.trim();
2154021544
return {
@@ -21544,12 +21548,13 @@ function loggingDiagnostics(env = process.env) {
2154421548
maxStringChars: maxStringChars(env),
2154521549
logFile: logFile || void 0,
2154621550
logFileMaxBytes: logFile ? logFileMaxBytes(env) : void 0,
21547-
logFileLastError: lastLogFileError
21551+
logFileLastError: lastLogFileError,
21552+
stderrMirrorDisabled
2154821553
};
2154921554
}
2155021555
function writeDefaultLog(line) {
2155121556
try {
21552-
if (!process.stderr.destroyed && process.stderr.writable) {
21557+
if (!stderrMirrorDisabled && !process.stderr.destroyed && process.stderr.writable) {
2155321558
process.stderr.write(`${line}
2155421559
`, (error2) => {
2155521560
if (error2) lastLogFileError = error2.message;
@@ -23308,6 +23313,75 @@ var CodexJobManager = class {
2330823313
};
2330923314
var jobManager = new CodexJobManager();
2331023315

23316+
// src/processes.ts
23317+
import { execFile } from "node:child_process";
23318+
import { promisify } from "node:util";
23319+
var execFileAsync = promisify(execFile);
23320+
var staleCpuThresholdPct = 25;
23321+
function sanitizeCommand(command) {
23322+
const redacted = redactSensitiveText(command);
23323+
return redacted.length <= 500 ? redacted : `${redacted.slice(0, 500)}...`;
23324+
}
23325+
function parsePsProcessLine(line) {
23326+
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+([0-9.]+)\s+(\S+)\s+(.+)$/);
23327+
if (!match) return void 0;
23328+
const [, pid, ppid, pgid, stat3, cpuPct, elapsed, command] = match;
23329+
if (!pid || !ppid || !pgid || !stat3 || !cpuPct || !elapsed || !command) return void 0;
23330+
return {
23331+
pid: Number(pid),
23332+
ppid: Number(ppid),
23333+
pgid: Number(pgid),
23334+
stat: stat3,
23335+
cpuPct: Number(cpuPct),
23336+
elapsed,
23337+
command: sanitizeCommand(command)
23338+
};
23339+
}
23340+
function isPluginProcess(command) {
23341+
return command.includes("codex-subagents") && command.includes("dist/index.js");
23342+
}
23343+
function pluginProcessDiagnosticsFromSnapshots(snapshots, currentPid = process.pid) {
23344+
const pluginProcesses = snapshots.filter((snapshot3) => isPluginProcess(snapshot3.command));
23345+
const staleSuspects = pluginProcesses.filter((snapshot3) => snapshot3.pid !== currentPid && snapshot3.ppid === 1);
23346+
return {
23347+
supported: true,
23348+
currentPid,
23349+
pluginProcesses,
23350+
staleSuspects,
23351+
highCpuStaleSuspects: staleSuspects.filter((snapshot3) => snapshot3.cpuPct >= staleCpuThresholdPct)
23352+
};
23353+
}
23354+
async function detectPluginProcesses() {
23355+
if (process.platform === "win32") {
23356+
return {
23357+
supported: false,
23358+
currentPid: process.pid,
23359+
pluginProcesses: [],
23360+
staleSuspects: [],
23361+
highCpuStaleSuspects: [],
23362+
error: "process scan is not implemented on Windows"
23363+
};
23364+
}
23365+
try {
23366+
const { stdout } = await execFileAsync(
23367+
"ps",
23368+
["-axo", "pid=,ppid=,pgid=,stat=,%cpu=,etime=,command="],
23369+
{ timeout: 1e3, maxBuffer: 1e6, encoding: "utf8" }
23370+
);
23371+
const snapshots = stdout.split("\n").map((line) => parsePsProcessLine(line)).filter((snapshot3) => Boolean(snapshot3));
23372+
return pluginProcessDiagnosticsFromSnapshots(snapshots);
23373+
} catch (error2) {
23374+
return {
23375+
supported: false,
23376+
currentPid: process.pid,
23377+
pluginProcesses: [],
23378+
staleSuspects: [],
23379+
highCpuStaleSuspects: [],
23380+
error: error2 instanceof Error ? error2.message : String(error2)
23381+
};
23382+
}
23383+
}
23384+
2331123385
// src/recovery.ts
2331223386
function messageFor(error2) {
2331323387
return error2 instanceof Error ? error2.message : String(error2);
@@ -23523,7 +23597,20 @@ function isBrokenStdioError(error2) {
2352323597
const value = error2;
2352423598
if (typeof value.code === "string" && brokenStdioCodes.has(value.code)) return true;
2352523599
const message = typeof value.message === "string" ? value.message.toLowerCase() : "";
23526-
return message.includes("broken pipe") || message.includes("stream has been destroyed") || message.includes("write after end");
23600+
return message.includes("broken pipe") || message.includes("channel closed") || message.includes("socket closed") || message.includes("stream has been destroyed") || message.includes("write after end");
23601+
}
23602+
function isOrphanedParentPid(parentPid) {
23603+
return parentPid <= 1;
23604+
}
23605+
function updateOrphanWatchdogState(options) {
23606+
if (!isOrphanedParentPid(options.parentPid)) {
23607+
return { shouldExit: false };
23608+
}
23609+
const orphanSinceMs = options.previousOrphanSinceMs ?? options.nowMs;
23610+
return {
23611+
orphanSinceMs,
23612+
shouldExit: options.nowMs - orphanSinceMs >= options.graceMs
23613+
};
2352723614
}
2352823615

2352923616
// src/response.ts
@@ -26333,35 +26420,41 @@ async function loggedToolCall(tool, args, extra, run) {
2633326420
return errorResult(error2, tool);
2633426421
}
2633526422
}
26336-
function installTransportLogging(transport, shutdown) {
26423+
function installTransportLogging(transport, shutdown, isShuttingDown) {
2633726424
const previousOnMessage = transport.onmessage;
2633826425
transport.onmessage = (message) => {
26339-
logger.rawDebug("mcp.transport.inbound", {
26340-
message: summarizeRawTrafficForLog(message)
26341-
});
26426+
if (!isShuttingDown()) {
26427+
logger.rawDebug("mcp.transport.inbound", {
26428+
message: summarizeRawTrafficForLog(message)
26429+
});
26430+
}
2634226431
previousOnMessage?.(message);
2634326432
};
2634426433
const previousOnError = transport.onerror;
2634526434
transport.onerror = (error2) => {
2634626435
if (isBrokenStdioError(error2)) {
26347-
shutdown("mcp_transport_broken_stdio", 0, 500);
26436+
shutdown("mcp_transport_broken_stdio", 0, 250);
2634826437
return;
2634926438
}
26439+
if (isShuttingDown()) return;
2635026440
logger.error("mcp.transport.error", { error: errorForLog(error2) });
2635126441
previousOnError?.(error2);
2635226442
};
2635326443
const send = transport.send.bind(transport);
2635426444
transport.send = async (message) => {
26355-
logger.rawDebug("mcp.transport.outbound", {
26356-
message: summarizeRawTrafficForLog(message)
26357-
});
26445+
if (!isShuttingDown()) {
26446+
logger.rawDebug("mcp.transport.outbound", {
26447+
message: summarizeRawTrafficForLog(message)
26448+
});
26449+
}
2635826450
try {
2635926451
await send(message);
2636026452
} catch (error2) {
2636126453
if (isBrokenStdioError(error2)) {
26362-
shutdown("mcp_transport_send_broken_stdio", 0, 500);
26454+
shutdown("mcp_transport_send_broken_stdio", 0, 250);
2636326455
return;
2636426456
}
26457+
if (isShuttingDown()) return;
2636526458
logger.error("mcp.transport.send_failed", { error: errorForLog(error2) });
2636626459
throw error2;
2636726460
}
@@ -26604,6 +26697,7 @@ async function mapWithConcurrency(items, maxParallel, worker) {
2660426697
}
2660526698
async function codexStatusPayload(codexBin) {
2660626699
const status = await probeCodexVersion(codexBin);
26700+
const processes = await detectPluginProcesses();
2660726701
return {
2660826702
ok: !status.error,
2660926703
binary: status.binary,
@@ -26643,6 +26737,7 @@ async function codexStatusPayload(codexBin) {
2664326737
sessions: sessionManager.stats(),
2664426738
logging: loggingDiagnostics(),
2664526739
artifacts: outputArtifactDiagnostics(),
26740+
processes,
2664626741
diagnostics: {
2664726742
...diagnosticStats(),
2664826743
recentFailures: recentDiagnosticEvents(20)
@@ -26700,6 +26795,14 @@ async function codexDoctorPayload(args = {}) {
2670026795
checks.push({ name: "artifacts", ok: true, detail: outputArtifactDiagnostics() });
2670126796
checks.push({ name: "diagnostics", ok: true, detail: diagnosticStats() });
2670226797
checks.push({ name: "lifecycle", ok: true, detail: lifecycleStats() });
26798+
const processes = await detectPluginProcesses();
26799+
const processOk = processes.highCpuStaleSuspects.length === 0;
26800+
if (!processOk) ok = false;
26801+
checks.push({
26802+
name: "stale_processes",
26803+
ok: processOk,
26804+
detail: processes
26805+
});
2670326806
return {
2670426807
ok,
2670526808
checks,
@@ -28609,12 +28712,55 @@ registerCleanupHandler(async (reason) => {
2860928712
jobManager.cancelAll(reason);
2861028713
await sessionManager.shutdown(reason);
2861128714
});
28715+
function envBoundedInteger(name, fallback, min, max) {
28716+
const parsed = Number(process.env[name]);
28717+
if (!Number.isInteger(parsed) || parsed <= 0) return fallback;
28718+
return Math.max(min, Math.min(parsed, max));
28719+
}
28720+
function installOrphanWatchdog(controller) {
28721+
const forceOrphanForTest = process.env.CODEX_SUBAGENTS_TEST_FORCE_ORPHAN === "1";
28722+
const initialParentPid = forceOrphanForTest ? 2 : process.ppid;
28723+
if (initialParentPid <= 1) {
28724+
logger.warn("lifecycle.orphan_watchdog.disabled", {
28725+
parentPid: initialParentPid,
28726+
reason: "process started without a live parent"
28727+
});
28728+
return;
28729+
}
28730+
const intervalMs = envBoundedInteger("CODEX_SUBAGENTS_ORPHAN_WATCHDOG_INTERVAL_MS", 1e3, 100, 6e4);
28731+
const graceMs = envBoundedInteger("CODEX_SUBAGENTS_ORPHAN_WATCHDOG_GRACE_MS", 2e3, 250, 6e4);
28732+
let orphanSinceMs;
28733+
const interval = setInterval(() => {
28734+
if (controller.isShuttingDown()) return;
28735+
const parentPid = forceOrphanForTest ? 1 : process.ppid;
28736+
const state = updateOrphanWatchdogState({
28737+
parentPid,
28738+
nowMs: Date.now(),
28739+
previousOrphanSinceMs: orphanSinceMs,
28740+
graceMs
28741+
});
28742+
orphanSinceMs = state.orphanSinceMs;
28743+
if (state.shouldExit) {
28744+
logger.warn("lifecycle.orphaned_parent", {
28745+
parentPid,
28746+
graceMs,
28747+
reason: "daemonless stdio MCP server lost its parent process"
28748+
});
28749+
controller.shutdown("orphaned_parent", 0, 250);
28750+
}
28751+
}, intervalMs);
28752+
interval.unref();
28753+
registerCleanupHandler(() => clearInterval(interval));
28754+
}
2861228755
function installProcessCleanup() {
2861328756
let shutdownStarted = false;
28757+
let requestedExitCode;
2861428758
const shutdown = (reason, exitCode, graceMs = 2500) => {
2861528759
if (shutdownStarted) return;
2861628760
shutdownStarted = true;
28617-
const forceExit = exitCode === void 0 ? void 0 : setTimeout(() => process.exit(exitCode), Math.max(1e3, graceMs + 1e3));
28761+
requestedExitCode = exitCode;
28762+
disableStderrLogMirrorForShutdown();
28763+
const forceExit = exitCode === void 0 ? void 0 : setTimeout(() => process.exit(exitCode), Math.max(250, graceMs + 250));
2861828764
forceExit?.unref();
2861928765
void cleanupRuntime(reason, graceMs).finally(() => {
2862028766
if (forceExit) clearTimeout(forceExit);
@@ -28623,42 +28769,60 @@ function installProcessCleanup() {
2862328769
};
2862428770
const shutdownOnBrokenStdio = (reason) => (error2) => {
2862528771
if (isBrokenStdioError(error2)) {
28626-
shutdown(reason, 0, 500);
28772+
shutdown(reason, 0, 250);
2862728773
return;
2862828774
}
2862928775
logger.error(`${reason}.error`, { error: errorForLog(error2) });
2863028776
};
2863128777
process.once("SIGINT", () => shutdown("SIGINT", 130));
2863228778
process.once("SIGTERM", () => shutdown("SIGTERM", 143));
28633-
process.stdin.once("close", () => shutdown("stdin_close", 0, 500));
28634-
process.stdin.once("end", () => shutdown("stdin_end", 0, 500));
28779+
process.stdin.once("close", () => shutdown("stdin_close", 0, 250));
28780+
process.stdin.once("end", () => shutdown("stdin_end", 0, 250));
2863528781
process.stdin.once("error", shutdownOnBrokenStdio("stdin"));
2863628782
process.stdout.once("error", shutdownOnBrokenStdio("stdout"));
2863728783
process.stderr.once("error", shutdownOnBrokenStdio("stderr"));
28638-
return shutdown;
28784+
return {
28785+
shutdown,
28786+
isShuttingDown: () => shutdownStarted,
28787+
exitCode: () => requestedExitCode
28788+
};
2863928789
}
2864028790
async function main() {
28641-
const shutdown = installProcessCleanup();
28791+
const shutdownController = installProcessCleanup();
28792+
installOrphanWatchdog(shutdownController);
2864228793
process.on("unhandledRejection", (error2) => {
28794+
if (shutdownController.isShuttingDown()) return;
2864328795
if (isBrokenStdioError(error2)) {
28644-
shutdown("unhandled_broken_stdio", 0, 500);
28796+
shutdownController.shutdown("unhandled_broken_stdio", 0, 250);
2864528797
return;
2864628798
}
2864728799
logger.error("process.unhandled_rejection", { error: errorForLog(error2) });
2864828800
});
2864928801
process.on("uncaughtException", (error2) => {
28802+
if (shutdownController.isShuttingDown()) {
28803+
process.exit(shutdownController.exitCode() ?? 1);
28804+
}
2865028805
if (isBrokenStdioError(error2)) {
28651-
shutdown("uncaught_broken_stdio", 0, 500);
28806+
shutdownController.shutdown("uncaught_broken_stdio", 0, 250);
2865228807
return;
2865328808
}
2865428809
logger.error("process.uncaught_exception", { error: errorForLog(error2) });
28655-
shutdown("uncaught_exception", 1, 500);
28810+
shutdownController.shutdown("uncaught_exception", 1, 500);
2865628811
});
2865728812
logger.info("server.starting", {
2865828813
logging: loggingDiagnostics()
2865928814
});
2866028815
const transport = new StdioServerTransport();
28661-
installTransportLogging(transport, shutdown);
28816+
registerCleanupHandler(async () => {
28817+
try {
28818+
await transport.close();
28819+
} catch (error2) {
28820+
if (!isBrokenStdioError(error2)) {
28821+
logger.error("mcp.transport.close_failed", { error: errorForLog(error2) });
28822+
}
28823+
}
28824+
});
28825+
installTransportLogging(transport, shutdownController.shutdown, shutdownController.isShuttingDown);
2866228826
await server.connect(transport);
2866328827
logger.info("server.connected", { transport: "stdio" });
2866428828
}

0 commit comments

Comments
 (0)