diff --git a/e2e/desktop-packaged/stale-launchd-registration.test.ts b/e2e/desktop-packaged/stale-launchd-registration.test.ts new file mode 100644 index 000000000..d7f6c3b99 --- /dev/null +++ b/e2e/desktop-packaged/stale-launchd-registration.test.ts @@ -0,0 +1,885 @@ +/* oxlint-disable executor/no-try-catch-or-throw, executor/no-error-constructor, executor/no-promise-reject -- e2e boundary: drives launchd, a packaged Electron process, CDP, and optional ffmpeg recording */ +import { type ChildProcess, execFile, execFileSync, spawn } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "@effect/vitest"; +import { Effect } from "effect"; + +import { scenario } from "../src/scenario"; +import { RunDir } from "../src/services"; + +const execFileAsync = promisify(execFile); +const SERVICE_LABEL = "sh.executor.daemon"; +const SCENARIO_NAME = + "Desktop packaged supervised attach: stale launchd registration pins the 15s dead boot wait"; +const RECORDING_FILE = resolve("recordings/stale-launchd-slow-boot.mp4"); +const RECORDING_WIDTH = 1280; +const RECORDING_HEIGHT = 720; +const STALE_PLIST_PORT = 58_251; + +interface PackagedExecutorBridge { + readonly getServerConnection: () => Promise<{ + readonly kind: string; + readonly origin: string; + } | null>; +} + +interface PackagedApp { + readonly child: ChildProcess; + cdp: CdpPage; + readonly debugPort: string; + readonly output: () => string; +} + +interface CdpResponse { + readonly id: number; + readonly result?: T; + readonly error?: { readonly message?: string; readonly data?: string }; +} + +interface CdpEvaluateResult { + readonly result: { readonly value?: unknown }; + readonly exceptionDetails?: unknown; +} + +interface CdpTarget { + readonly type: string; + readonly url: string; + readonly webSocketDebuggerUrl?: string; +} + +interface ServerManifest { + readonly kind: string; + readonly pid: number; +} + +interface LaunchTiming { + readonly app: PackagedApp; + readonly pageCdpMs: number; +} + +interface BootResult { + readonly pageCdpMs: number; + readonly sidecarReadyMs: number; + readonly connectionKind: string; + readonly origin: string; + readonly manifestKind: string; + readonly manifestPid: number; + readonly matchedLogLine: string | null; +} + +declare global { + interface Window { + readonly executor: PackagedExecutorBridge; + } +} + +class CdpPage { + private nextId = 1; + private readonly pending = new Map< + number, + { + readonly resolve: (value: unknown) => void; + readonly reject: (error: Error) => void; + } + >(); + + private constructor(private readonly socket: WebSocket) { + socket.addEventListener("message", (event) => { + const data = event.data; + if (typeof data !== "string") return; + const message = JSON.parse(data) as CdpResponse; + if (!message.id) return; + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.error) { + pending.reject(new Error(message.error.message ?? "CDP command failed")); + return; + } + pending.resolve(message.result); + }); + socket.addEventListener("close", () => { + for (const [, pending] of this.pending) { + pending.reject(new Error("CDP socket closed")); + } + this.pending.clear(); + }); + } + + static connect = (url: string): Promise => + new Promise((resolveConnect, rejectConnect) => { + const socket = new WebSocket(url); + const timer = setTimeout(() => { + socket.close(); + rejectConnect(new Error(`Timed out connecting to page CDP target ${url}`)); + }, 30_000); + socket.addEventListener( + "open", + () => { + clearTimeout(timer); + resolveConnect(new CdpPage(socket)); + }, + { once: true }, + ); + socket.addEventListener( + "error", + () => { + clearTimeout(timer); + rejectConnect(new Error(`Failed to connect to page CDP target ${url}`)); + }, + { once: true }, + ); + }); + + command = async (method: string, params: Record = {}): Promise => { + const id = this.nextId; + this.nextId += 1; + const result = new Promise((resolveCommand, rejectCommand) => { + this.pending.set(id, { + resolve: (value) => resolveCommand(value as T), + reject: rejectCommand, + }); + }); + this.socket.send(JSON.stringify({ id, method, params })); + return result; + }; + + evaluate = async (expression: string): Promise => { + const result = await this.command("Runtime.evaluate", { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (result.exceptionDetails) { + throw new Error(`CDP evaluation failed: ${JSON.stringify(result.exceptionDetails)}`); + } + return result.result.value as T; + }; + + waitForText = async (text: string, timeoutMs: number): Promise => { + const deadline = Date.now() + timeoutMs; + const expression = `document.body?.innerText.includes(${JSON.stringify(text)}) ?? false`; + for (;;) { + if (await this.evaluate(expression).catch(() => false)) return; + if (Date.now() >= deadline) throw new Error(`Timed out waiting for text: ${text}`); + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); + } + }; + + setViewport = async (width: number, height: number): Promise => { + await this.command("Emulation.setDeviceMetricsOverride", { + width, + height, + deviceScaleFactor: 1, + mobile: false, + }); + }; + + screenshot = async (path: string): Promise => { + const result = await this.command<{ readonly data: string }>("Page.captureScreenshot", { + format: "png", + fromSurface: true, + }); + writeFileSync(path, Buffer.from(result.data, "base64")); + }; + + close = (): void => { + this.socket.close(); + }; +} + +const appExe = process.env.E2E_DESKTOP_APP_EXE; +const executorBin = process.env.E2E_DESKTOP_EXECUTOR_BIN; + +const macAquaAvailable = (): boolean => { + if (process.platform !== "darwin") return false; + try { + return execFileSync("launchctl", ["managername"], { encoding: "utf8" }).trim() === "Aqua"; + } catch { + return false; + } +}; + +const packagedSingleInstanceAvailable = (): boolean => { + if (process.platform !== "darwin" || !appExe) return true; + try { + const lines = execFileSync("pgrep", ["-fl", "Executor.app/Contents/MacOS/Executor"], { + encoding: "utf8", + }) + .split("\n") + .filter(Boolean); + return !lines.some((line) => !line.includes(appExe)); + } catch { + return true; + } +}; + +const requireBundle = (): { readonly app: string; readonly executor: string } => { + if (!appExe || !executorBin) { + throw new Error( + "E2E_DESKTOP_APP_EXE / E2E_DESKTOP_EXECUTOR_BIN not set, did desktop-packaged.globalsetup run?", + ); + } + return { app: appExe, executor: executorBin }; +}; + +const currentUid = (): number => { + const getuid = (process as { readonly getuid?: () => number }).getuid; + return typeof getuid === "function" ? getuid.call(process) : 0; +}; + +const serviceTarget = (): string => `gui/${currentUid()}/${SERVICE_LABEL}`; +const launchAgentPath = (): string => + join(homedir(), "Library", "LaunchAgents", `${SERVICE_LABEL}.plist`); +const isolatedDesktopSettingsDir = (home: string): string => + join(home, ".executor-desktop-settings"); + +const packagedAppEnv = (home: string): NodeJS.ProcessEnv => ({ + ...process.env, + HOME: home, + EXECUTOR_DESKTOP_SETTINGS_DIR: isolatedDesktopSettingsDir(home), +}); + +const cleanBootEnv = (home: string): NodeJS.ProcessEnv => ({ + ...packagedAppEnv(home), + EXECUTOR_TEST_SKIP_BACKGROUND_SERVICE: "1", + EXECUTOR_TEST_AUTO_CONFIRM_BACKGROUND_SERVICE: "0", +}); + +interface LaunchdServiceSnapshot { + readonly plist: string | null; + readonly wasLoaded: boolean; +} + +const launchctl = async (args: ReadonlyArray): Promise => { + try { + await execFileAsync("launchctl", [...args]); + return true; + } catch { + return false; + } +}; + +const captureLaunchdService = (): LaunchdServiceSnapshot | null => { + if (process.platform !== "darwin") return null; + const path = launchAgentPath(); + const plist = existsSync(path) ? readFileSync(path, "utf8") : null; + let wasLoaded = false; + try { + execFileSync("launchctl", ["print", serviceTarget()], { stdio: "ignore" }); + wasLoaded = true; + } catch { + wasLoaded = false; + } + return { plist, wasLoaded }; +}; + +const restoreLaunchdService = async (snapshot: LaunchdServiceSnapshot | null): Promise => { + if (!snapshot) return; + const target = serviceTarget(); + await launchctl(["bootout", target]); + const path = launchAgentPath(); + if (snapshot.plist === null) { + rmSync(path, { force: true }); + return; + } + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, snapshot.plist, { mode: 0o600 }); + chmodSync(path, 0o600); + await launchctl(["enable", target]); + if (snapshot.wasLoaded) { + const bootstrapped = await launchctl(["bootstrap", `gui/${currentUid()}`, path]); + if (bootstrapped) await launchctl(["kickstart", "-k", target]); + } +}; + +const waitForPageWebSocket = async (debugPort: string): Promise => { + const deadline = Date.now() + 120_000; + for (;;) { + const targets = (await fetch(`http://127.0.0.1:${debugPort}/json/list`) + .then((response) => (response.ok ? response.json() : [])) + .catch(() => [])) as ReadonlyArray; + const page = targets.find( + (target) => + target.type === "page" && + target.webSocketDebuggerUrl && + !target.url.startsWith("devtools://"), + ); + if (page?.webSocketDebuggerUrl) return page.webSocketDebuggerUrl; + if (Date.now() >= deadline) { + throw new Error("Timed out waiting for packaged app page CDP target"); + } + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); + } +}; + +const launchPackaged = async ( + env: NodeJS.ProcessEnv, + recorder: BootRecorder | null, +): Promise => { + const { app } = requireBundle(); + const startedAt = Date.now(); + let output = ""; + let settled = false; + const child = spawn(app, ["--remote-debugging-port=0"], { + env, + stdio: ["ignore", "pipe", "pipe"], + }); + recorder?.start(startedAt); + + try { + const browserCdpUrl = await new Promise((resolveLaunch, rejectLaunch) => { + const timer = setTimeout(() => { + rejectLaunch(new Error(`Timed out waiting for packaged app CDP URL\n${output}`)); + }, 120_000); + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + fn(); + }; + const collectOutput = (chunk: Buffer) => { + const text = chunk.toString(); + output = (output + text).slice(-16_384); + const match = output.match(/DevTools listening on (ws:\/\/[^\s]+)/); + if (match) settle(() => resolveLaunch(match[1])); + }; + child.stdout?.on("data", collectOutput); + child.stderr?.on("data", collectOutput); + child.once("error", (error) => settle(() => rejectLaunch(error))); + child.once("exit", (code, signal) => + settle(() => + rejectLaunch( + new Error(`Packaged app exited before CDP (code=${code} signal=${signal})\n${output}`), + ), + ), + ); + }); + + const debugPort = new URL(browserCdpUrl).port; + const pageCdpUrl = await waitForPageWebSocket(debugPort); + const cdp = await CdpPage.connect(pageCdpUrl); + await cdp.command("Runtime.enable"); + await cdp.command("Page.enable"); + await cdp.setViewport(RECORDING_WIDTH, RECORDING_HEIGHT); + recorder?.attachPage(cdp); + return { + app: { child, cdp, debugPort, output: () => output }, + pageCdpMs: Date.now() - startedAt, + }; + } catch (error) { + await stopProcess(child); + throw error; + } +}; + +const stopProcess = async (child: ChildProcess | undefined): Promise => { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolveStop) => { + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + resolveStop(); + }, 5_000); + child.once("exit", () => { + clearTimeout(timeout); + resolveStop(); + }); + child.kill("SIGTERM"); + }); +}; + +const closePackaged = async (app: PackagedApp | undefined): Promise => { + app?.cdp.close(); + await stopProcess(app?.child); +}; + +const xmlEscape = (value: string): string => + value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll(">", ">") + .replaceAll("<", "<"); + +const writeStaleLaunchAgentPlist = (home: string): void => { + const { executor } = requireBundle(); + const dataDir = join(home, ".executor"); + const logs = join(dataDir, "logs"); + const plistDir = join(home, "Library", "LaunchAgents"); + mkdirSync(plistDir, { recursive: true }); + mkdirSync(logs, { recursive: true }); + mkdirSync(dataDir, { recursive: true }); + + const programArguments = [ + executor, + "daemon", + "run", + "--foreground", + "--port", + String(STALE_PLIST_PORT), + "--hostname", + "127.0.0.1", + ]; + const programArgs = programArguments + .map((arg) => ` ${xmlEscape(arg)}`) + .join("\n"); + const plist = ` + + + + Label + ${SERVICE_LABEL} + ProgramArguments + +${programArgs} + + EnvironmentVariables + + EXECUTOR_SUPERVISED + 1 + EXECUTOR_DATA_DIR + ${xmlEscape(dataDir)} + EXECUTOR_SCOPE_DIR + ${xmlEscape(dataDir)} + EXECUTOR_CLIENT + desktop + + RunAtLoad + + KeepAlive + + SuccessfulExit + + + ProcessType + Background + WorkingDirectory + ${xmlEscape(dataDir)} + StandardOutPath + ${xmlEscape(join(logs, "daemon.log"))} + StandardErrorPath + ${xmlEscape(join(logs, "daemon.error.log"))} + + +`; + writeFileSync(join(plistDir, `${SERVICE_LABEL}.plist`), plist, { mode: 0o600 }); +}; + +const readServerManifest = (home: string): ServerManifest => { + const manifestPath = join(home, ".executor", "server-control", "server.json"); + return JSON.parse(readFileSync(manifestPath, "utf8")) as ServerManifest; +}; + +const readKickstartFailureLine = (home: string): string | null => { + const logPath = join(home, "Library", "Logs", "Executor", "main.log"); + if (!existsSync(logPath)) return null; + return ( + readFileSync(logPath, "utf8") + .split("\n") + .find( + (line) => + line.includes("launchctl kickstart failed (exit 113)") && + line.includes('Could not find service "sh.executor.daemon"'), + ) ?? null + ); +}; + +const waitForKickstartFailureLine = async (home: string): Promise => { + const deadline = Date.now() + 30_000; + for (;;) { + const line = readKickstartFailureLine(home); + if (line) return line; + if (Date.now() >= deadline) { + const logPath = join(home, "Library", "Logs", "Executor", "main.log"); + const tail = existsSync(logPath) + ? readFileSync(logPath, "utf8").split("\n").slice(-20).join("\n") + : ""; + throw new Error(`Timed out waiting for kickstart failure log line\n${tail}`); + } + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); + } +}; + +const bootPackagedApp = async (input: { + readonly home: string; + readonly env: NodeJS.ProcessEnv; + readonly recorder: BootRecorder | null; + readonly expectKickstartFailure: boolean; +}): Promise => { + let app: PackagedApp | undefined; + const startedAt = Date.now(); + try { + const launched = await launchPackaged(input.env, input.recorder); + app = launched.app; + const page = app.cdp; + await page.waitForText("Settings", 120_000); + const sidecarReadyMs = Date.now() - startedAt; + await input.recorder?.captureReadyFrame(); + await input.recorder?.captureReadyFrame(); + const connection = await page.evaluate<{ + readonly kind: string; + readonly origin: string; + } | null>("window.executor.getServerConnection()"); + expect(connection, "the app eventually exposes a server connection").not.toBeNull(); + const manifest = readServerManifest(input.home); + const matchedLogLine = input.expectKickstartFailure + ? await waitForKickstartFailureLine(input.home) + : readKickstartFailureLine(input.home); + return { + pageCdpMs: launched.pageCdpMs, + sidecarReadyMs, + connectionKind: connection!.kind, + origin: connection!.origin, + manifestKind: manifest.kind, + manifestPid: manifest.pid, + matchedLogLine, + }; + } finally { + await input.recorder?.stop(); + await closePackaged(app); + } +}; + +const commandAvailable = (command: string): boolean => { + try { + execFileSync("which", [command], { stdio: "ignore" }); + return true; + } catch { + return false; + } +}; + +const emitEvidence = (line: string): void => { + execFileSync("printf", ["%s\n", line], { stdio: "inherit" }); +}; + +const FONT: Record = { + " ": ["00000", "00000", "00000", "00000", "00000", "00000", "00000"], + ".": ["00000", "00000", "00000", "00000", "00000", "01100", "01100"], + "0": ["11110", "10010", "10010", "10010", "10010", "10010", "11110"], + "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"], + "2": ["11110", "00010", "00010", "11110", "10000", "10000", "11110"], + "3": ["11110", "00010", "00010", "01110", "00010", "00010", "11110"], + "4": ["10010", "10010", "10010", "11110", "00010", "00010", "00010"], + "5": ["11110", "10000", "10000", "11110", "00010", "00010", "11110"], + "6": ["11110", "10000", "10000", "11110", "10010", "10010", "11110"], + "7": ["11110", "00010", "00010", "00100", "00100", "01000", "01000"], + "8": ["11110", "10010", "10010", "11110", "10010", "10010", "11110"], + "9": ["11110", "10010", "10010", "11110", "00010", "00010", "11110"], + A: ["01110", "10001", "10001", "11111", "10001", "10001", "10001"], + B: ["11110", "10001", "10001", "11110", "10001", "10001", "11110"], + C: ["01111", "10000", "10000", "10000", "10000", "10000", "01111"], + D: ["11110", "10001", "10001", "10001", "10001", "10001", "11110"], + E: ["11111", "10000", "10000", "11110", "10000", "10000", "11111"], + F: ["11111", "10000", "10000", "11110", "10000", "10000", "10000"], + G: ["01111", "10000", "10000", "10011", "10001", "10001", "01111"], + H: ["10001", "10001", "10001", "11111", "10001", "10001", "10001"], + I: ["11111", "00100", "00100", "00100", "00100", "00100", "11111"], + J: ["00111", "00010", "00010", "00010", "10010", "10010", "01100"], + K: ["10001", "10010", "10100", "11000", "10100", "10010", "10001"], + L: ["10000", "10000", "10000", "10000", "10000", "10000", "11111"], + M: ["10001", "11011", "10101", "10101", "10001", "10001", "10001"], + N: ["10001", "11001", "10101", "10011", "10001", "10001", "10001"], + O: ["01110", "10001", "10001", "10001", "10001", "10001", "01110"], + P: ["11110", "10001", "10001", "11110", "10000", "10000", "10000"], + Q: ["01110", "10001", "10001", "10001", "10101", "10010", "01101"], + R: ["11110", "10001", "10001", "11110", "10100", "10010", "10001"], + S: ["01111", "10000", "10000", "01110", "00001", "00001", "11110"], + T: ["11111", "00100", "00100", "00100", "00100", "00100", "00100"], + U: ["10001", "10001", "10001", "10001", "10001", "10001", "01110"], + V: ["10001", "10001", "10001", "10001", "10001", "01010", "00100"], + W: ["10001", "10001", "10001", "10101", "10101", "10101", "01010"], + X: ["10001", "10001", "01010", "00100", "01010", "10001", "10001"], + Y: ["10001", "10001", "01010", "00100", "00100", "00100", "00100"], + Z: ["11111", "00001", "00010", "00100", "01000", "10000", "11111"], +}; + +const putPixel = (pixels: Buffer, width: number, x: number, y: number, rgb: readonly number[]) => { + const index = (y * width + x) * 3; + pixels[index] = rgb[0]; + pixels[index + 1] = rgb[1]; + pixels[index + 2] = rgb[2]; +}; + +const fillRect = ( + pixels: Buffer, + width: number, + x: number, + y: number, + rectWidth: number, + rectHeight: number, + rgb: readonly number[], +): void => { + for (let yy = y; yy < y + rectHeight; yy += 1) { + for (let xx = x; xx < x + rectWidth; xx += 1) { + putPixel(pixels, width, xx, yy, rgb); + } + } +}; + +const writePpm = (path: string, width: number, height: number, pixels: Buffer): void => { + writeFileSync(path, Buffer.concat([Buffer.from(`P6\n${width} ${height}\n255\n`), pixels])); +}; + +const makeLabelPpm = (path: string, label: string, height: number): void => { + const pixels = Buffer.alloc(RECORDING_WIDTH * height * 3); + fillRect(pixels, RECORDING_WIDTH, 0, 0, RECORDING_WIDTH, height, [17, 18, 23]); + fillRect(pixels, RECORDING_WIDTH, 0, 0, RECORDING_WIDTH, 110, [0, 0, 0]); + const scale = 6; + let x = 34; + const y = 34; + for (const char of label.toUpperCase()) { + const glyph = FONT[char] ?? FONT[" "]; + for (let row = 0; row < glyph.length; row += 1) { + for (let col = 0; col < glyph[row].length; col += 1) { + if (glyph[row][col] === "1") { + fillRect( + pixels, + RECORDING_WIDTH, + x + col * scale, + y + row * scale, + scale, + scale, + [255, 255, 255], + ); + } + } + } + x += 6 * scale; + if (x > RECORDING_WIDTH - 40) break; + } + writePpm(path, RECORDING_WIDTH, height, pixels); +}; + +class BootRecorder { + private frame = 0; + private page: CdpPage | null = null; + private startedAt = 0; + private timer: ReturnType | null = null; + private queue: Promise = Promise.resolve(); + private enabled: boolean; + + constructor( + private readonly frameDir: string, + private readonly outputPath: string, + ) { + this.enabled = process.env.E2E_RECORD === "1" && commandAvailable("ffmpeg"); + } + + static create = (runDir: string): BootRecorder | null => { + if (process.env.E2E_RECORD !== "1") return null; + const frameDir = join(runDir, "stale-launchd-slow-boot-frames"); + rmSync(frameDir, { recursive: true, force: true }); + mkdirSync(frameDir, { recursive: true }); + mkdirSync(dirname(RECORDING_FILE), { recursive: true }); + rmSync(RECORDING_FILE, { force: true }); + return new BootRecorder(frameDir, RECORDING_FILE); + }; + + start = (startedAt: number): void => { + this.startedAt = startedAt; + if (!this.enabled) return; + this.enqueueFrame("waiting for Electron window", null); + this.timer = setInterval(() => { + this.enqueueFrame( + this.page ? "app window reachable" : "waiting for Electron window", + this.page, + ); + }, 1_000); + }; + + attachPage = (page: CdpPage): void => { + this.page = page; + this.enqueueFrame("app window reachable", page); + }; + + captureReadyFrame = async (): Promise => { + if (!this.enabled) return; + this.enqueueFrame("managed sidecar ready", this.page); + await this.queue; + }; + + stop = async (): Promise => { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + if (!this.enabled) return; + await this.queue; + if (this.frame > 0) { + execFileSync( + "ffmpeg", + [ + "-y", + "-framerate", + "1", + "-i", + join(this.frameDir, "frame-%04d.ppm"), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + this.outputPath, + ], + { stdio: "ignore" }, + ); + } + }; + + recordingStats = (): { + readonly path: string; + readonly size: number; + readonly frames: number; + } | null => { + if (!existsSync(this.outputPath)) return null; + return { path: this.outputPath, size: statSync(this.outputPath).size, frames: this.frame }; + }; + + private enqueueFrame = (state: string, page: CdpPage | null): void => { + if (!this.enabled) return; + this.queue = this.queue + .then(() => this.writeFrame(state, page)) + .catch((error) => { + this.enabled = false; + console.warn(`[stale-launchd-repro] recording disabled after frame error: ${error}`); + }); + }; + + private writeFrame = async (state: string, page: CdpPage | null): Promise => { + this.frame += 1; + const elapsedSeconds = (Date.now() - this.startedAt) / 1000; + const output = join(this.frameDir, `frame-${String(this.frame).padStart(4, "0")}.ppm`); + const label = `${elapsedSeconds.toFixed(1)}s ${state}`; + if (!page) { + makeLabelPpm(output, label, RECORDING_HEIGHT); + return; + } + const raw = join(this.frameDir, `raw-${String(this.frame).padStart(4, "0")}.png`); + const labelFrame = join(this.frameDir, `label-${String(this.frame).padStart(4, "0")}.ppm`); + await page.screenshot(raw); + makeLabelPpm(labelFrame, label, 110); + execFileSync( + "ffmpeg", + [ + "-y", + "-i", + raw, + "-i", + labelFrame, + "-filter_complex", + `[0:v]scale=${RECORDING_WIDTH}:${RECORDING_HEIGHT}:force_original_aspect_ratio=decrease,pad=${RECORDING_WIDTH}:${RECORDING_HEIGHT}:(ow-iw)/2:(oh-ih)/2:color=black[base];[base][1:v]overlay=0:0`, + "-frames:v", + "1", + output, + ], + { stdio: "ignore" }, + ); + rmSync(raw, { force: true }); + rmSync(labelFrame, { force: true }); + }; +} + +const run = async (runDir: string): Promise => { + const cleanHome = mkdtempSync(join(tmpdir(), "executor-pkg-clean-boot-control-")); + const staleHome = mkdtempSync(join(tmpdir(), "executor-pkg-stale-launchd-")); + const launchdSnapshot = captureLaunchdService(); + const staleRecorder = BootRecorder.create(runDir); + + try { + const clean = await bootPackagedApp({ + home: cleanHome, + env: cleanBootEnv(cleanHome), + recorder: null, + expectKickstartFailure: false, + }); + + writeStaleLaunchAgentPlist(staleHome); + await launchctl(["bootout", serviceTarget()]); + const stale = await bootPackagedApp({ + home: staleHome, + env: packagedAppEnv(staleHome), + recorder: staleRecorder, + expectKickstartFailure: true, + }); + + /* + * Regression pin for the current broken behavior: + * a stale plist makes `service status` report registered even though launchd + * cannot find the label. The packaged app logs the kickstart failure, then + * still pays the full 15s wait before falling back to managed-spawn. + * + * When the fix lands, invert the >=14s stale assertions. The desired + * behavior is a fast-fail to managed-spawn, close to the clean control. + */ + expect(stale.matchedLogLine, "stale boot logs the launchd kickstart failure").toContain( + "launchctl kickstart failed (exit 113)", + ); + expect(stale.matchedLogLine, "stale boot says launchd cannot find the label").toContain( + 'Could not find service "sh.executor.daemon"', + ); + expect( + stale.pageCdpMs, + "window/CDP is not reachable until the dead wait is paid", + ).toBeGreaterThanOrEqual(14_000); + expect( + stale.sidecarReadyMs, + "managed sidecar readiness is delayed by the dead wait", + ).toBeGreaterThanOrEqual(14_000); + expect(stale.connectionKind, "the app eventually recovers through managed-spawn").toBe( + "desktop-sidecar", + ); + expect( + new URL(stale.origin).port, + "managed recovery must not be a launchd attach to the stale plist port", + ).not.toBe(String(STALE_PLIST_PORT)); + expect( + clean.sidecarReadyMs, + "the clean control skips supervision and boots quickly", + ).toBeLessThan(10_000); + expect( + stale.sidecarReadyMs, + "the stale launchd path is dramatically slower than the clean control", + ).toBeGreaterThan(clean.sidecarReadyMs + 5_000); + + const recording = staleRecorder?.recordingStats(); + emitEvidence( + `[stale-launchd-repro] clean=${clean.sidecarReadyMs}ms stale=${stale.sidecarReadyMs}ms stalePageCdp=${stale.pageCdpMs}ms cleanOrigin=${clean.origin} staleOrigin=${stale.origin} staleConnectionKind=${stale.connectionKind} staleManifestKind=${stale.manifestKind} stalePid=${stale.manifestPid}`, + ); + emitEvidence(`[stale-launchd-repro] matchedLogLine=${stale.matchedLogLine}`); + if (recording) { + emitEvidence( + `[stale-launchd-repro] recording=${recording.path} frames=${recording.frames} size=${recording.size}`, + ); + } + } finally { + await restoreLaunchdService(launchdSnapshot); + rmSync(cleanHome, { recursive: true, force: true }); + rmSync(staleHome, { recursive: true, force: true }); + } +}; + +if (!macAquaAvailable() || !packagedSingleInstanceAvailable()) { + it.skip(`${SCENARIO_NAME} (needs macOS Aqua and no already-running Executor.app)`, () => {}); +} else { + scenario( + SCENARIO_NAME, + { timeout: 360_000 }, + Effect.gen(function* () { + requireBundle(); + const runDir = yield* RunDir; + yield* Effect.promise(() => run(runDir)); + }), + ); +} diff --git a/e2e/recordings/stale-launchd-slow-boot.mp4 b/e2e/recordings/stale-launchd-slow-boot.mp4 new file mode 100644 index 000000000..15c25f91b Binary files /dev/null and b/e2e/recordings/stale-launchd-slow-boot.mp4 differ