🤖 perf: shard OpenSSH exec paths and simplify SSH project sync (#3125)

## Summary
This keeps the SSH scaling wins that matter in practice — sharded
OpenSSH exec paths, serialized same-project sync/import work, hashed
remote project layout, and persisted SSH workspace roots where they
still match a known checkout shape — while deleting the explicit
master-pool/coordinator machinery and replacing most mock-heavy runtime
unit suites with higher-level SSH integration coverage.

## Background
The original branch tackled real SSH bottlenecks: one implicit control
socket per host, duplicate same-project bundle imports, and
basename-derived remote roots that could collide. But the first
implementation grew into an app-managed OpenSSH master scheduler with a
large amount of lifecycle, compatibility, and mock-test surface area.
This revision deliberately simplifies the design so the branch still
improves SSH scalability without carrying most of that maintenance
burden.

## Implementation
- shard short-lived OpenSSH exec/file traffic across a small
deterministic set of `ControlPath`s via `ControlMaster=auto`, keep
`sshConnectionPool` as the single bootstrap/backoff layer, re-check
requested shard readiness after waiting on another inflight probe, and
cap any follow-up shard probe to the remaining acquire budget
- simplify `SSHRuntime` to one hashed remote project layout, one
serialized per-project sync path, one current-snapshot marker, and
ref-manifest validation before snapshot reuse
- keep bundle imports in `refs/mux-bundle/*` and UUID bundle temp paths,
but remove the explicit `openSshMasterPool`, `projectSyncCoordinator`,
remote branch-metadata persistence, and hot-path legacy layout
auto-detection layers
- preserve persisted SSH workspace roots only when config still points
at a known canonical or legacy checkout shape, including sibling
multi-project path hints for repo operations and resolving an upgraded
worktree's actual common git dir before rename/delete worktree commands
- replace the runtime-internal unit suites with focused Docker-backed
SSH integration coverage for concurrent exec bursts, rename/delete
lifecycle, snapshot recovery, and checked-out branch collision cases,
plus targeted `sshConnectionPool` and legacy-worktree regressions

## Validation
- `make typecheck`
- `make static-check`
- `bun test ./src/node/runtime/runtimeHelpers.test.ts`
- `bun test ./src/node/runtime/sshConnectionPool.test.ts`
- `bun test src/node/services/workspaceProjectRepos.test.ts
src/node/services/workspaceService.test.ts
src/node/services/workspaceService.multiProject.test.ts`
- `TEST_INTEGRATION=1 bun x jest tests/runtime/runtime.test.ts
--runInBand --testNamePattern='SSHRuntime'`
- `TEST_INTEGRATION=1 bun x jest tests/runtime/runtime.test.ts
--runInBand --testNamePattern='legacy base repo for upgraded SSH
worktrees'`

## Risks
Moderate. The simplified transport no longer has the same theoretical
ceiling as the previous explicit master-pool design, but it still
improves on the original baseline by removing the single implicit
host-wide control path from short-lived SSH execs. The main
compatibility edge is that SSH workspaces rooted outside the known
canonical or legacy checkout shapes now fall back to the canonical
hashed layout and may need re-init instead of transparent path
inference.

## Pains
The original explicit-pool direction accumulated a lot of policy and
mock-test code. The cleanup work here was mostly about deleting that
surface area without giving back the core scaling fixes, which in turn
meant shifting confidence into the real Docker-backed SSH integration
harness.

---

_Generated with `mux` • Model: `openai:gpt-5.4` • Thinking: `xhigh` •
Cost: `$269.00`_

<!-- mux-attribution: model=openai:gpt-5.4 thinking=xhigh costs=269.00
-->

Author: ammar-agent

src/node/runtime/DockerRuntime.ts

@@ -353,7 +353,10 @@ export class DockerRuntime extends RemoteRuntime {
     return `cd ${shescape.quote(cwd)}`;
   }
 
-  protected spawnRemoteProcess(fullCommand: string, _options: ExecOptions): Promise<SpawnResult> {
+  protected spawnRemoteProcess(
+    fullCommand: string,
+    _options: ExecOptions & { deadlineMs?: number }
+  ): Promise<SpawnResult> {
     // Verify container name is available
     if (!this.containerName) {
       throw new RuntimeError(

src/node/runtime/RemoteRuntime.ts

@@ -11,7 +11,6 @@
  * - spawnRemoteProcess() - how to spawn the external process (ssh/docker)
  * - getBasePath() - base directory for workspace operations
  * - quoteForRemote() - path quoting strategy
- * - onExitCode() - optional exit code handling (SSH connection pool)
  */
 
 import type { ChildProcess } from "child_process";
@@ -45,8 +44,12 @@ import { getAtomicWriteTempPath } from "./atomicWriteTempPath";
 export interface SpawnResult {
   /** The spawned child process */
   process: ChildProcess;
-  /** Optional async work to do before exec (e.g., acquire connection) */
-  preExec?: Promise<void>;
+  /** Optional transport-scoped exit handling (e.g., master-pool health accounting). */
+  onExit?: (exitCode: number, stderr: string) => void;
+  /** Optional close handling that must run even for synthetic abort/timeout exits. */
+  onClose?: () => void;
+  /** Optional transport-scoped spawn error handling. */
+  onError?: (error: Error) => void;
 }
 
 /**
@@ -59,11 +62,11 @@ export abstract class RemoteRuntime implements Runtime {
    *
    * @param fullCommand The full shell command to execute (already wrapped in bash -c)
    * @param options Original exec options
-   * @returns The spawned process and optional pre-exec work
+   * @returns The spawned process and optional transport lifecycle hooks
    */
   protected abstract spawnRemoteProcess(
     fullCommand: string,
-    options: ExecOptions
+    options: ExecOptions & { deadlineMs?: number }
   ): Promise<SpawnResult>;
 
   /**
@@ -84,15 +87,6 @@ export abstract class RemoteRuntime implements Runtime {
    */
   protected abstract cdCommand(cwd: string): string;
 
-  /**
-   * Called when exec completes with an exit code.
-   * Subclasses can use this for connection pool health tracking.
-   * @param stderr - Captured stderr for error reporting (e.g., SSH connection failures)
-   */
-  protected onExitCode(_exitCode: number, _options: ExecOptions, _stderr: string): void {
-    // Default: no-op. SSH overrides to report to connection pool.
-  }
-
   /**
    * Command prefix (e.g., "SSH" or "Docker") for logging.
    */
@@ -138,8 +132,13 @@ export abstract class RemoteRuntime implements Runtime {
     }
 
     // Spawn the remote process (SSH or Docker)
-    // For SSH, this awaits connection pool backoff before spawning
-    const { process: childProcess } = await this.spawnRemoteProcess(fullCommand, options);
+    const timeoutMs = options.timeout !== undefined ? options.timeout * 1000 : undefined;
+    const deadlineMs = timeoutMs !== undefined ? Date.now() + timeoutMs : undefined;
+    const spawnResult = await this.spawnRemoteProcess(fullCommand, {
+      ...options,
+      deadlineMs,
+    });
+    const { process: childProcess } = spawnResult;
 
     // Short-lived commands can close stdin before writes/close complete.
     if (childProcess.stdin) {
@@ -163,23 +162,23 @@ export abstract class RemoteRuntime implements Runtime {
     // Create promises for exit code and duration immediately.
     const exitCode = new Promise<number>((resolve, reject) => {
       childProcess.on("close", (code, signal) => {
-        if (aborted || options.abortSignal?.aborted) {
-          resolve(EXIT_CODE_ABORTED);
-          return;
-        }
-        if (timedOut) {
-          resolve(EXIT_CODE_TIMEOUT);
-          return;
+        const finalExitCode =
+          aborted || options.abortSignal?.aborted
+            ? EXIT_CODE_ABORTED
+            : timedOut
+              ? EXIT_CODE_TIMEOUT
+              : (code ?? (signal ? -1 : 0));
+
+        if (finalExitCode !== EXIT_CODE_ABORTED && finalExitCode !== EXIT_CODE_TIMEOUT) {
+          spawnResult.onExit?.(finalExitCode, stderrForErrorReporting);
         }
-        const finalExitCode = code ?? (signal ? -1 : 0);
-
-        // Let subclass handle exit code (e.g., SSH connection pool)
-        this.onExitCode(finalExitCode, options, stderrForErrorReporting);
+        spawnResult.onClose?.();
 
         resolve(finalExitCode);
       });
 
       childProcess.on("error", (err) => {
+        spawnResult.onError?.(err);
         reject(
           new RuntimeError(
             `Failed to execute ${this.commandPrefix} command: ${err.message}`,
@@ -226,8 +225,10 @@ export abstract class RemoteRuntime implements Runtime {
       void exitCode.finally(() => abortSignal.removeEventListener("abort", onAbort));
     }
 
-    // Handle timeout
-    if (options.timeout !== undefined) {
+    // Handle timeout. Include connection acquisition time in the local deadline so
+    // user-configured timeouts do not silently stretch while the runtime waits for SSH capacity.
+    if (timeoutMs !== undefined) {
+      const remainingTimeoutMs = Math.max(0, (deadlineMs ?? Date.now()) - Date.now());
       const timeoutHandle = setTimeout(() => {
         timedOut = true;
 
@@ -245,7 +246,7 @@ export abstract class RemoteRuntime implements Runtime {
           disposable[Symbol.dispose]();
         }, 1000);
         hardKillHandle.unref();
-      }, options.timeout * 1000);
+      }, remainingTimeoutMs);
 
       void exitCode.finally(() => clearTimeout(timeoutHandle));
     }