Merge pull request #1039 from heygen-com/fix/orphaned-child-processes

fix: clean up orphaned Chrome/ffmpeg on preview exit

Author: miguel-heygen

packages/cli/src/commands/preview.ts

@@ -28,6 +28,7 @@ import {
   killActiveServers,
   type FindPortResult,
 } from "../server/portUtils.js";
+import { killOrphanedProcesses, killProcessTree } from "../utils/orphanCleanup.js";
 
 export default defineCommand({
   meta: { name: "preview", description: "Start the studio for previewing compositions" },
@@ -96,6 +97,14 @@ export default defineCommand({
       return;
     }
 
+    // Kill orphaned chrome-headless-shell processes from previous crashed sessions.
+    const orphansKilled = killOrphanedProcesses();
+    if (orphansKilled > 0) {
+      console.log(
+        `  ${c.dim(`Cleaned up ${orphansKilled} orphaned process${orphansKilled === 1 ? "" : "es"} from a previous session.`)}`,
+      );
+    }
+
     const rawArg = args.dir;
     const dir = resolve(rawArg ?? ".");
 
@@ -249,8 +258,18 @@ async function runDevMode(
     });
   }
 
-  // Wait for child to exit. Ctrl+C sends SIGINT to the entire process group,
-  // so the child (Vite) receives it directly — no need to intercept or forward.
+  // Kill the child's entire process tree on SIGTERM/SIGINT. Ctrl+C sends
+  // SIGINT to the foreground process group (covers the common case), but
+  // `kill <pid>` only targets this process — the child tree (Vite + Chrome)
+  // would survive without explicit cleanup.
+  // On Windows, killProcessTree is a no-op (pgrep/ps unavailable); Ctrl+C
+  // propagates via the console process group instead.
+  const shutdown = () => {
+    if (child.pid) killProcessTree(child.pid);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+
   return new Promise<void>((resolve) => {
     child.on("close", () => resolve());
   });
@@ -349,6 +368,13 @@ async function runLocalStudioMode(
     });
   }
 
+  // Same tree-kill handler as dev mode. No-op on Windows (see comment above).
+  const shutdown = () => {
+    if (child.pid) killProcessTree(child.pid);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+
   return new Promise<void>((resolve) => {
     child.on("close", () => resolve());
   });
@@ -477,21 +503,42 @@ async function runEmbeddedMode(
       shuttingDown = true;
       process.off("SIGINT", shutdown);
       process.off("SIGTERM", shutdown);
-      // Close the readline interface so a second Ctrl+C during the grace
-      // period below doesn't re-emit SIGINT and trigger Node's default
-      // exit-130 behaviour, contradicting our intent to exit cleanly.
       rl?.close();
-      // `server.close()` can take a second or two to drain keep-alive
-      // connections; surface progress so the terminal doesn't look frozen.
       console.log();
       console.log(`  ${c.dim("Shutting down studio...")}`);
-      result.server.close(() => resolveRun());
-      // If close() hangs on an open connection, force exit after a short
-      // grace period. Exit 0 because user-initiated Ctrl+C isn't an error
-      // — a non-zero code makes pnpm / npm print ELIFECYCLE.
-      setTimeout(() => process.exit(0), 2000).unref();
+
+      // Hard deadline: if cleanup hangs (e.g. dead Chrome never responds to
+      // browser.close()), force exit. Armed before awaiting cleanup so it
+      // can't be blocked by a stuck drainBrowserPool().
+      setTimeout(() => process.exit(0), 3000).unref();
+
+      // Kill ffmpeg first (sync, fast), then drain browsers (async, slower).
+      const cleanup = async () => {
+        const { closeThumbnailBrowser } = await import("../server/studioServer.js");
+        const { drainBrowserPool, killTrackedProcesses } = await import("@hyperframes/engine");
+        killTrackedProcesses();
+        await closeThumbnailBrowser().catch(() => {});
+        await drainBrowserPool().catch(() => {});
+      };
+
+      cleanup()
+        .catch(() => {})
+        .finally(() => {
+          result.server.close(() => resolveRun());
+        });
     };
     process.once("SIGINT", shutdown);
     process.once("SIGTERM", shutdown);
+
+    // Last-resort cleanup for crash paths (unhandled exceptions/rejections)
+    // that bypass the signal handlers. Eagerly resolve the sync killer so
+    // the 'exit' handler (which is synchronous) can call it directly.
+    import("@hyperframes/engine")
+      .then(({ killTrackedProcesses }) => {
+        process.once("exit", () => {
+          if (!shuttingDown) killTrackedProcesses();
+        });
+      })
+      .catch(() => {});
   });
 }

packages/cli/src/server/studioServer.ts

@@ -148,16 +148,6 @@ async function getThumbnailBrowser(): Promise<import("puppeteer-core").Browser |
         _thumbnailBrowser = null;
         _thumbnailBrowserInitializing = null;
       });
-      // Release the pool ref on process exit so the browser closes cleanly.
-      const onExit = async () => {
-        const { releaseBrowser } = await import("@hyperframes/engine");
-        if (_thumbnailBrowser) {
-          await releaseBrowser(_thumbnailBrowser).catch(() => {});
-          _thumbnailBrowser = null;
-        }
-      };
-      process.once("SIGTERM", () => void onExit());
-      process.once("SIGINT", () => void onExit());
       return _thumbnailBrowser;
     } catch (err) {
       console.warn(
@@ -172,6 +162,15 @@ async function getThumbnailBrowser(): Promise<import("puppeteer-core").Browser |
   return _thumbnailBrowserInitializing;
 }
 
+export async function closeThumbnailBrowser(): Promise<void> {
+  if (!_thumbnailBrowser) return;
+  const browser = _thumbnailBrowser;
+  _thumbnailBrowser = null;
+  _thumbnailBrowserInitializing = null;
+  const { releaseBrowser } = await import("@hyperframes/engine");
+  await releaseBrowser(browser).catch(() => {});
+}
+
 // ── Server factory ──────────────────────────────────────────────────────────
 
 export interface StudioServerOptions {

packages/cli/src/utils/orphanCleanup.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect } from "vitest";
+import { spawn } from "node:child_process";
+import { killProcessTree, killOrphanedProcesses } from "./orphanCleanup.js";
+
+const IS_UNIX = process.platform !== "win32";
+
+describe.skipIf(!IS_UNIX)("killProcessTree", () => {
+  it("kills a process and all its children", async () => {
+    // Spawn a parent that spawns two sleeping children
+    const parent = spawn("bash", ["-c", "sleep 60 & sleep 60 & wait"], { stdio: "ignore" });
+    // Let children spawn
+    await new Promise((r) => setTimeout(r, 200));
+
+    const exitPromise = new Promise<void>((resolve) => parent.on("close", resolve));
+    killProcessTree(parent.pid!);
+
+    await exitPromise;
+
+    // Verify parent is dead
+    expect(() => process.kill(parent.pid!, 0)).toThrow();
+  }, 5000);
+
+  it("handles non-existent PID gracefully", () => {
+    // Should not throw for a PID that doesn't exist
+    killProcessTree(999999999);
+  });
+
+  it("escalates to SIGKILL after grace period", async () => {
+    // Spawn a process that traps SIGTERM
+    const proc = spawn("bash", ["-c", "trap '' TERM; sleep 60"], { stdio: "ignore" });
+    await new Promise((r) => setTimeout(r, 100));
+
+    const exitPromise = new Promise<void>((resolve) => proc.on("close", resolve));
+    killProcessTree(proc.pid!);
+
+    // Should die within 1s (500ms SIGKILL grace + buffer)
+    await exitPromise;
+    expect(() => process.kill(proc.pid!, 0)).toThrow();
+  }, 5000);
+});
+
+describe.skipIf(!IS_UNIX)("killOrphanedProcesses", () => {
+  it("returns 0 when no orphans exist", () => {
+    const killed = killOrphanedProcesses();
+    expect(killed).toBe(0);
+  });
+
+  it("does not kill non-orphaned Chrome processes", () => {
+    // Our current process is not an orphan (PPID !== 1), so any
+    // chrome-headless-shell processes we'd find with our PID as
+    // ancestor wouldn't be killed.
+    const killed = killOrphanedProcesses();
+    expect(killed).toBe(0);
+  });
+});

packages/cli/src/utils/orphanCleanup.ts

@@ -0,0 +1,137 @@
+import { execSync } from "node:child_process";
+
+/**
+ * Find and kill orphaned Chrome processes from previous crashed sessions.
+ * Targets both chrome-headless-shell (production/CI) and Google Chrome
+ * launched by Puppeteer (dev mode). Puppeteer Chrome is identified by the
+ * `puppeteer_dev_chrome_profile` marker in its user-data-dir argument.
+ *
+ * An orphan is a process whose PPID=1 (reparented to init/launchd after
+ * its parent died). We kill the orphan's entire subtree so child helper
+ * processes (GPU, renderer, network, etc.) are also cleaned up.
+ *
+ * Scoped to the current user via `pgrep -u` to avoid touching other
+ * users' processes on shared machines.
+ *
+ * Returns the count of killed process trees.
+ */
+export function killOrphanedProcesses(): number {
+  if (process.platform === "win32") return 0;
+
+  let killed = 0;
+
+  for (const name of ["chrome-headless-shell", "chrome_headless_shell"]) {
+    killed += killOrphansByName(name);
+  }
+
+  killed += killOrphansByName("puppeteer_dev_chrome_profile");
+
+  return killed;
+}
+
+/**
+ * Kill an entire process tree rooted at `pid`. Walks descendants
+ * depth-first so children are killed before parents, preventing
+ * re-adoption races.
+ *
+ * No-op on Windows — process groups are managed differently and
+ * the pgrep/ps utilities are not available.
+ */
+export function killProcessTree(pid: number, signal: NodeJS.Signals = "SIGTERM"): void {
+  if (process.platform === "win32") return;
+
+  const descendants = getDescendants(pid);
+  const allPids = [...descendants.reverse(), pid];
+
+  for (const p of allPids) {
+    try {
+      process.kill(p, signal);
+    } catch {
+      // Already exited.
+    }
+  }
+
+  // Escalate to SIGKILL after a short grace period for any survivors.
+  if (signal !== "SIGKILL") {
+    setTimeout(() => {
+      for (const p of allPids) {
+        try {
+          process.kill(p, "SIGKILL");
+        } catch {
+          // Already exited.
+        }
+      }
+    }, 500).unref();
+  }
+}
+
+function getDescendants(pid: number): number[] {
+  let children: number[];
+  try {
+    const raw = execSync(`pgrep -P ${pid}`, { encoding: "utf-8", timeout: 2000 }).trim();
+    if (!raw) return [];
+    children = raw
+      .split("\n")
+      .map((s) => parseInt(s, 10))
+      .filter((n) => !isNaN(n) && n > 0);
+  } catch {
+    return [];
+  }
+  const all: number[] = [];
+  for (const child of children) {
+    all.push(child);
+    all.push(...getDescendants(child));
+  }
+  return all;
+}
+
+function killOrphansByName(processName: string): number {
+  const uid = getUid();
+  const userFlag = uid !== null ? `-u ${uid} ` : "";
+  let pids: number[];
+  try {
+    const raw = execSync(`pgrep ${userFlag}-f ${processName}`, {
+      encoding: "utf-8",
+      timeout: 3000,
+    }).trim();
+    if (!raw) return 0;
+    pids = raw
+      .split("\n")
+      .map((s) => parseInt(s, 10))
+      .filter((n) => !isNaN(n) && n > 0);
+  } catch {
+    return 0;
+  }
+
+  let killed = 0;
+  for (const pid of pids) {
+    if (!isOrphan(pid)) continue;
+    killProcessTree(pid);
+    killed++;
+  }
+  return killed;
+}
+
+let _cachedUid: string | null | undefined;
+
+function getUid(): string | null {
+  if (_cachedUid !== undefined) return _cachedUid;
+  try {
+    _cachedUid = execSync("id -u", { encoding: "utf-8", timeout: 1000 }).trim();
+  } catch {
+    _cachedUid = null;
+  }
+  return _cachedUid;
+}
+
+function isOrphan(pid: number): boolean {
+  try {
+    const ppid = execSync(`ps -p ${pid} -o ppid=`, {
+      encoding: "utf-8",
+      timeout: 2000,
+    }).trim();
+    return ppid === "1";
+  } catch {
+    return false;
+  }
+}

packages/engine/src/index.ts

@@ -186,6 +186,8 @@ export {
   type RunFfmpegResult,
 } from "./utils/runFfmpeg.js";
 
+export { trackChildProcess, killTrackedProcesses } from "./utils/processTracker.js";
+
 export {
   decodePng,
   decodePngToRgb48le,

packages/engine/src/services/chunkEncoder.ts

@@ -8,6 +8,7 @@
 import { spawn } from "child_process";
 import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync, writeFileSync } from "fs";
 import { join, dirname } from "path";
+import { trackChildProcess } from "../utils/processTracker.js";
 import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
 import {
   type GpuEncoder,
@@ -404,6 +405,7 @@ export async function encodeFramesFromDir(
 
   return new Promise((resolve) => {
     const ffmpeg = spawn("ffmpeg", args);
+    trackChildProcess(ffmpeg);
     let stderr = "";
     const onAbort = () => {
       ffmpeg.kill("SIGTERM");
@@ -535,6 +537,7 @@ export async function encodeFramesChunkedConcat(
     const args = buildEncoderArgs(options, inputArgs, chunkPath, gpuEncoder);
     const chunkResult = await new Promise<{ success: boolean; error?: string }>((resolve) => {
       const ffmpeg = spawn("ffmpeg", args);
+      trackChildProcess(ffmpeg);
       let stderr = "";
       ffmpeg.stderr.on("data", (d) => {
         stderr += d.toString();
@@ -578,6 +581,7 @@ export async function encodeFramesChunkedConcat(
   ];
   const concatResult = await new Promise<{ success: boolean; error?: string }>((resolve) => {
     const ffmpeg = spawn("ffmpeg", concatArgs);
+    trackChildProcess(ffmpeg);
     let stderr = "";
     ffmpeg.stderr.on("data", (d) => {
       stderr += d.toString();

packages/engine/src/services/streamingEncoder.ts

@@ -13,6 +13,7 @@
  */
 
 import { spawn, type ChildProcess } from "child_process";
+import { trackChildProcess } from "../utils/processTracker.js";
 import { existsSync, mkdirSync, statSync } from "fs";
 import { dirname } from "path";
 
@@ -375,6 +376,7 @@ export async function spawnStreamingEncoder(
   const ffmpeg: ChildProcess = spawn("ffmpeg", args, {
     stdio: ["pipe", "pipe", "pipe"],
   });
+  trackChildProcess(ffmpeg);
 
   let exitStatus: "running" | "success" | "error" = "running";
   let stderr = "";