🤖 perf: shard OpenSSH masters and dedupe SSH project sync

- add an explicit OpenSSH master pool with sharded control sockets and lease-based exec reuse
- hash remote project layouts and move workspace metadata/snapshot markers to project-scoped files
- dedupe per-project bundle sync work and update unit plus SSH integration coverage for the new layout

---

_Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` • Cost: `$36.24`_

<!-- mux-attribution: model=openai:gpt-5.4 thinking=xhigh costs=36.24 -->

Author: ammar-agent

src/node/runtime/DockerRuntime.ts

@@ -353,7 +353,10 @@ export class DockerRuntime extends RemoteRuntime {
     return `cd ${shescape.quote(cwd)}`;
   }
 
-  protected spawnRemoteProcess(fullCommand: string, _options: ExecOptions): Promise<SpawnResult> {
+  protected spawnRemoteProcess(
+    fullCommand: string,
+    _options: ExecOptions & { deadlineMs?: number }
+  ): Promise<SpawnResult> {
     // Verify container name is available
     if (!this.containerName) {
       throw new RuntimeError(

src/node/runtime/RemoteRuntime.ts

@@ -47,6 +47,10 @@ export interface SpawnResult {
   process: ChildProcess;
   /** Optional async work to do before exec (e.g., acquire connection) */
   preExec?: Promise<void>;
+  /** Optional transport-scoped exit handling (e.g., master-pool health accounting). */
+  onExit?: (exitCode: number, stderr: string) => void;
+  /** Optional transport-scoped spawn error handling. */
+  onError?: (error: Error) => void;
 }
 
 /**
@@ -63,7 +67,7 @@ export abstract class RemoteRuntime implements Runtime {
    */
   protected abstract spawnRemoteProcess(
     fullCommand: string,
-    options: ExecOptions
+    options: ExecOptions & { deadlineMs?: number }
   ): Promise<SpawnResult>;
 
   /**
@@ -138,8 +142,13 @@ export abstract class RemoteRuntime implements Runtime {
     }
 
     // Spawn the remote process (SSH or Docker)
-    // For SSH, this awaits connection pool backoff before spawning
-    const { process: childProcess } = await this.spawnRemoteProcess(fullCommand, options);
+    const timeoutMs = options.timeout !== undefined ? options.timeout * 1000 : undefined;
+    const deadlineMs = timeoutMs !== undefined ? Date.now() + timeoutMs : undefined;
+    const spawnResult = await this.spawnRemoteProcess(fullCommand, {
+      ...options,
+      deadlineMs,
+    });
+    const { process: childProcess } = spawnResult;
 
     // Short-lived commands can close stdin before writes/close complete.
     if (childProcess.stdin) {
@@ -163,23 +172,22 @@ export abstract class RemoteRuntime implements Runtime {
     // Create promises for exit code and duration immediately.
     const exitCode = new Promise<number>((resolve, reject) => {
       childProcess.on("close", (code, signal) => {
-        if (aborted || options.abortSignal?.aborted) {
-          resolve(EXIT_CODE_ABORTED);
-          return;
-        }
-        if (timedOut) {
-          resolve(EXIT_CODE_TIMEOUT);
-          return;
-        }
-        const finalExitCode = code ?? (signal ? -1 : 0);
-
+        const finalExitCode =
+          aborted || options.abortSignal?.aborted
+            ? EXIT_CODE_ABORTED
+            : timedOut
+              ? EXIT_CODE_TIMEOUT
+              : (code ?? (signal ? -1 : 0));
+
+        spawnResult.onExit?.(finalExitCode, stderrForErrorReporting);
         // Let subclass handle exit code (e.g., SSH connection pool)
         this.onExitCode(finalExitCode, options, stderrForErrorReporting);
 
         resolve(finalExitCode);
       });
 
       childProcess.on("error", (err) => {
+        spawnResult.onError?.(err);
         reject(
           new RuntimeError(
             `Failed to execute ${this.commandPrefix} command: ${err.message}`,
@@ -226,8 +234,10 @@ export abstract class RemoteRuntime implements Runtime {
       void exitCode.finally(() => abortSignal.removeEventListener("abort", onAbort));
     }
 
-    // Handle timeout
-    if (options.timeout !== undefined) {
+    // Handle timeout. Include connection acquisition time in the local deadline so
+    // user-configured timeouts do not silently stretch while the runtime waits for SSH capacity.
+    if (timeoutMs !== undefined) {
+      const remainingTimeoutMs = Math.max(0, (deadlineMs ?? Date.now()) - Date.now());
       const timeoutHandle = setTimeout(() => {
         timedOut = true;
 
@@ -245,7 +255,7 @@ export abstract class RemoteRuntime implements Runtime {
           disposable[Symbol.dispose]();
         }, 1000);
         hardKillHandle.unref();
-      }, options.timeout * 1000);
+      }, remainingTimeoutMs);
 
       void exitCode.finally(() => clearTimeout(timeoutHandle));
     }

src/node/runtime/SSHRuntime.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it, beforeEach, afterEach, spyOn } from "bun:test";
 import * as runtimeHelpers from "@/node/utils/runtime/helpers";
 import { SSHRuntime, computeBaseRepoPath } from "./SSHRuntime";
+import { buildRemoteProjectLayout, getRemoteWorkspacePath } from "./remoteProjectLayout";
 import { createSSHTransport } from "./transports";
 
 /**
@@ -149,10 +150,8 @@ describe("SSHRuntime base repo config normalization", () => {
 });
 
 describe("SSHRuntime.createWorkspace", () => {
-  it("uses directoryName for the workspace path while preparing the remote parent directory", async () => {
-    const config = { host: "example.com", srcBaseDir: "/home/user/src" };
-    const runtime = new SSHRuntime(config, createSSHTransport(config, false));
-    const execSpy = spyOn(runtime, "exec").mockResolvedValue({
+  function createExecStream(exitCode = 0) {
+    return {
       stdout: new ReadableStream<Uint8Array>({
         start(controller) {
           controller.close();
@@ -164,18 +163,29 @@ describe("SSHRuntime.createWorkspace", () => {
         },
       }),
       stdin: new WritableStream<Uint8Array>(),
-      exitCode: Promise.resolve(0),
+      exitCode: Promise.resolve(exitCode),
       duration: Promise.resolve(0),
-    });
-    const readFileSpy = spyOn(runtime, "readFile").mockReturnValue(
-      new ReadableStream<Uint8Array>({
-        start(controller) {
-          controller.error(new Error("missing branch map"));
-        },
-      })
+    };
+  }
+
+  it("uses directoryName for the workspace path while preparing the remote parent directory", async () => {
+    const config = { host: "example.com", srcBaseDir: "/home/user/src" };
+    const runtime = new SSHRuntime(config, createSSHTransport(config, false));
+    const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo");
+    const expectedWorkspacePath = getRemoteWorkspacePath(expectedLayout, "review-slot");
+    const execSpy = spyOn(runtime, "exec").mockImplementation(() =>
+      Promise.resolve(createExecStream())
+    );
+    const readFileSpy = spyOn(runtime, "readFile").mockImplementation(
+      () =>
+        new ReadableStream<Uint8Array>({
+          start(controller) {
+            controller.error(new Error("missing branch metadata"));
+          },
+        })
     );
-    const writeFileSpy = spyOn(runtime, "writeFile").mockReturnValue(
-      new WritableStream<Uint8Array>()
+    const writeFileSpy = spyOn(runtime, "writeFile").mockImplementation(
+      () => new WritableStream<Uint8Array>()
     );
 
     try {
@@ -194,13 +204,16 @@ describe("SSHRuntime.createWorkspace", () => {
 
       expect(result).toEqual({
         success: true,
-        workspacePath: "/home/user/src/demo/review-slot",
-      });
-      expect(execSpy).toHaveBeenCalledWith('mkdir -p "/home/user/src/demo"', {
-        cwd: "/tmp",
-        timeout: 10,
-        abortSignal: undefined,
+        workspacePath: expectedWorkspacePath,
       });
+      expect(execSpy).toHaveBeenCalledWith(
+        `mkdir -p ${JSON.stringify(expectedLayout.projectRoot)}`,
+        {
+          cwd: "/tmp",
+          timeout: 10,
+          abortSignal: undefined,
+        }
+      );
     } finally {
       execSpy.mockRestore();
       readFileSpy.mockRestore();
@@ -223,6 +236,8 @@ describe("SSHRuntime.deleteWorkspace", () => {
   it("deletes the mapped workspace branch instead of the current remote checkout", async () => {
     const config = { host: "example.com", srcBaseDir: "/home/user/src" };
     const runtime = new SSHRuntime(config, createSSHTransport(config, false));
+    const expectedLayout = buildRemoteProjectLayout(config.srcBaseDir, "/projects/demo");
+    const expectedDeletedPath = getRemoteWorkspacePath(expectedLayout, "review-slot");
     const execSpy = spyOn(runtime, "exec").mockImplementation((command) => {
       if (command.includes("git diff --quiet") || command.includes("test -d")) {
         return Promise.resolve(createExecStream(0));
@@ -232,13 +247,14 @@ describe("SSHRuntime.deleteWorkspace", () => {
       }
       throw new Error(`Unexpected exec command: ${command}`);
     });
-    const readFileSpy = spyOn(runtime, "readFile").mockReturnValue(
-      new ReadableStream<Uint8Array>({
-        start(controller) {
-          controller.enqueue(new TextEncoder().encode('{"review-slot":"feature-branch"}\n'));
-          controller.close();
-        },
-      })
+    const readFileSpy = spyOn(runtime, "readFile").mockImplementation(
+      () =>
+        new ReadableStream<Uint8Array>({
+          start(controller) {
+            controller.enqueue(new TextEncoder().encode('{"review-slot":"feature-branch"}\n'));
+            controller.close();
+          },
+        })
     );
     const execBufferedSpy = spyOn(runtimeHelpers, "execBuffered").mockImplementation(
       (_runtime, command) => {
@@ -261,7 +277,7 @@ describe("SSHRuntime.deleteWorkspace", () => {
       const result = await runtime.deleteWorkspace("/projects/demo", "review-slot", true);
       expect(result).toEqual({
         success: true,
-        deletedPath: "/home/user/src/demo/review-slot",
+        deletedPath: expectedDeletedPath,
       });
     } finally {
       execSpy.mockRestore();
@@ -396,14 +412,16 @@ describe("SSHRuntime.resolvePath", () => {
 });
 describe("computeBaseRepoPath", () => {
   it("computes the correct bare repo path", () => {
-    // computeBaseRepoPath uses getProjectName (basename) to compute:
-    // <srcBaseDir>/<projectName>/.mux-base.git
+    const layout = buildRemoteProjectLayout("~/mux", "/Users/me/code/my-project");
     const result = computeBaseRepoPath("~/mux", "/Users/me/code/my-project");
-    expect(result).toBe("~/mux/my-project/.mux-base.git");
+    expect(result).toBe(layout.baseRepoPath);
+    expect(result).toMatch(/^~\/mux\/my-project-[a-f0-9]{12}\/\.mux-base\.git$/);
   });
 
   it("handles absolute srcBaseDir", () => {
+    const layout = buildRemoteProjectLayout("/home/user/src", "/code/repo");
     const result = computeBaseRepoPath("/home/user/src", "/code/repo");
-    expect(result).toBe("/home/user/src/repo/.mux-base.git");
+    expect(result).toBe(layout.baseRepoPath);
+    expect(result).toMatch(/^\/home\/user\/src\/repo-[a-f0-9]{12}\/\.mux-base\.git$/);
   });
 });