fix(docker): apply docker-host-path-prefix to all compose service mounts (#3218)

salmanmkc · web-flow · commit 385ec4d90fc5 · 2026-05-15T07:59:06.000-07:00
The agent and iptables-init containers coordinate via a shared bind-mounted
init-signal directory at /tmp/awf-init. The iptables-init container writes
ready/output.log there after running setup-iptables.sh, and the agent's
entrypoint waits for those files before continuing.

buildAgentVolumes() applies dockerHostPathPrefix to its mount sources so the
agent's /tmp/awf-init bind is daemon-resolvable on split runner/Docker daemon
filesystems (e.g. ARC + DinD). buildIptablesInitService() did not, so once
--docker-host-path-prefix was set the two containers bound to two different
daemon-side directories. The init container could complete successfully and
the agent would still time out after 30s with 'No init container output log
found' because its bind target stayed empty.

The same gap existed in the squid, api-proxy, and cli-proxy service builders:
their bind-mount sources (squid logs, SSL cert/key/db, api-proxy logs,
cli-proxy logs, optional DIFC CA cert) were never run through the prefix
translation, so on ARC/DinD their logs would land in daemon-local directories
and optional file mounts could fail when Docker auto-creates a directory at
the unstaged source path.

Extract normalize/translate/applyHostPathPrefixToVolumes into a shared
host-path-prefix module and call applyHostPathPrefixToVolumes() at the end
of every service builder's volume list construction. agent-volumes.ts
delegates to the shared helper and re-exports the helpers for backwards
compatibility. doh-proxy has no bind mounts and is unchanged.

Add a parameterized symmetric invariant test that walks every bind mount
on every compose service and asserts the prefix is applied uniformly when
set (and skipped otherwise), so any future service builder is protected
against the same class of asymmetric translation bug.
diff --git a/src/compose-generator.ts b/src/compose-generator.ts
@@ -147,6 +147,7 @@ export function generateDockerCompose(
     environment,
     networkConfig,
     initSignalDir,
+    dockerHostPathPrefix: config.dockerHostPathPrefix,
   });
 
   // ── Assemble base services ─────────────────────────────────────────────────
diff --git a/src/services/agent-service.test.ts b/src/services/agent-service.test.ts
@@ -132,6 +132,100 @@ describe('agent service', () => {
       expect(initService.restart).toBe('no');
     });
 
+    it('should mount init-signal dir without translation when dockerHostPathPrefix is unset', () => {
+      const result = generateDockerCompose(mockConfig, mockNetworkConfig);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const initService = result.services['iptables-init'] as any;
+      const volumes = initService.volumes as string[];
+
+      // Source path is the runner-side init-signal dir, container path is /tmp/awf-init
+      expect(volumes).toContain(`${mockConfig.workDir}/init-signal:/tmp/awf-init:rw`);
+    });
+
+    it('should apply dockerHostPathPrefix to the iptables-init init-signal volume', () => {
+      // Regression: when --docker-host-path-prefix is set (e.g. ARC + DinD), the agent
+      // container's init-signal mount source is prefixed via translateBindMountHostPath.
+      // The iptables-init container's mount source must be prefixed identically — otherwise
+      // the two containers bind to different daemon-side directories and the agent times
+      // out with "No init container output log found" because the ready file written by
+      // setup-iptables.sh lands in a different bind-mount target.
+      const configWithPrefix = {
+        ...mockConfig,
+        dockerHostPathPrefix: '/host',
+      };
+      const result = generateDockerCompose(configWithPrefix, mockNetworkConfig);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const initService = result.services['iptables-init'] as any;
+      const initVolumes = initService.volumes as string[];
+      const agentVolumes = result.services.agent.volumes as string[];
+
+      const expectedSource = `/host${mockConfig.workDir}/init-signal`;
+      expect(initVolumes).toContain(`${expectedSource}:/tmp/awf-init:rw`);
+
+      // The agent must mount the SAME daemon-side source so they share the ready file.
+      expect(agentVolumes).toContain(`${expectedSource}:/tmp/awf-init:rw`);
+    });
+
+    it('should normalize trailing slash in dockerHostPathPrefix for iptables-init mount', () => {
+      const configWithPrefix = {
+        ...mockConfig,
+        dockerHostPathPrefix: '/host/',
+      };
+      const result = generateDockerCompose(configWithPrefix, mockNetworkConfig);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const initService = result.services['iptables-init'] as any;
+      const initVolumes = initService.volumes as string[];
+
+      expect(initVolumes).toContain(`/host${mockConfig.workDir}/init-signal:/tmp/awf-init:rw`);
+    });
+
+    // Symmetric invariant: every absolute, non-kernel-virtual bind-mount source on every
+    // service must be prefixed when dockerHostPathPrefix is set. This catches the original
+    // class of bug (asymmetric translation between services that share a daemon-side dir)
+    // for any future service builder, not just iptables-init.
+    describe.each([
+      { name: 'unset', prefix: undefined as string | undefined, expectPrefixed: false },
+      { name: 'empty', prefix: '', expectPrefixed: false },
+      { name: 'whitespace', prefix: '   ', expectPrefixed: false },
+      { name: '/host', prefix: '/host', expectPrefixed: true },
+      { name: '/host/ (trailing slash)', prefix: '/host/', expectPrefixed: true },
+    ])('symmetric prefix translation across compose services (dockerHostPathPrefix=$name)', ({ prefix, expectPrefixed }) => {
+      it('every absolute, non-kernel-virtual bind-mount source is prefixed consistently', () => {
+        const cfg = {
+          ...mockConfig,
+          dockerHostPathPrefix: prefix,
+          // Exercise sibling services that also build bind mounts.
+          enableApiProxy: true,
+          difcProxyHost: 'proxy.example.com:18443',
+          difcProxyCaCert: '/etc/ssl/ca.crt',
+        };
+        const result = generateDockerCompose(cfg, mockNetworkConfig);
+
+        const allVolumes: string[] = [];
+        for (const [, svc] of Object.entries(result.services)) {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const volumes = (svc as any).volumes as string[] | undefined;
+          if (Array.isArray(volumes)) allVolumes.push(...volumes);
+        }
+
+        // Sanity: at least one workDir-derived mount on every service we touched
+        expect(allVolumes.some(v => v.includes(mockConfig.workDir))).toBe(true);
+
+        for (const mount of allVolumes) {
+          const [src] = mount.split(':');
+          // Skip relative sources (named volumes) and the kernel virtual / /dev/null exemptions
+          if (!src.startsWith('/')) continue;
+          if (src === '/dev/null' || src.startsWith('/dev') || src.startsWith('/sys') || src.startsWith('/proc')) continue;
+
+          if (expectPrefixed) {
+            expect(src).toMatch(/^\/host(\/|$)/);
+          } else {
+            expect(src).not.toMatch(/^\/host(\/|$)/);
+          }
+        }
+      });
+    });
+
     it('should apply container hardening measures', () => {
       const result = generateDockerCompose(mockConfig, mockNetworkConfig);
       const agent = result.services.agent;
diff --git a/src/services/agent-service.ts b/src/services/agent-service.ts
@@ -15,6 +15,7 @@ import { NetworkConfig, ImageBuildConfig } from './squid-service';
 // Re-export functions for backwards compatibility
 export { buildAgentEnvironment } from './agent-environment';
 export { buildAgentVolumes } from './agent-volumes';
+import { applyHostPathPrefixToVolumes } from './host-path-prefix';
 
 // ─── Agent Service ────────────────────────────────────────────────────────────
 
@@ -206,6 +207,12 @@ interface IptablesInitServiceParams {
   environment: Record<string, string>;
   networkConfig: NetworkConfig;
   initSignalDir: string;
+  // When the Docker daemon resolves bind-mount sources from a different filesystem
+  // view than the runner (e.g. ARC + DinD), translate the init-signal mount source
+  // through the same prefix used for agent volumes. Without this, the agent and
+  // iptables-init containers land on two different daemon-side directories and the
+  // ready/output.log handshake silently fails ("No init container output log found").
+  dockerHostPathPrefix?: string;
 }
 
 /**
@@ -214,7 +221,16 @@ interface IptablesInitServiceParams {
  * without ever granting NET_ADMIN to the agent itself.
  */
 export function buildIptablesInitService(params: IptablesInitServiceParams): any {
-  const { agentService, environment, networkConfig, initSignalDir } = params;
+  const { agentService, environment, networkConfig, initSignalDir, dockerHostPathPrefix } = params;
+
+  // The init-signal mount must use the same source path that the agent container uses,
+  // otherwise the two containers bind to different daemon-side directories and the
+  // ready-file handshake fails. buildAgentVolumes() applies dockerHostPathPrefix to its
+  // mounts, so do the same here via the shared helper.
+  const [initSignalMount] = applyHostPathPrefixToVolumes(
+    [`${initSignalDir}:/tmp/awf-init:rw`],
+    dockerHostPathPrefix,
+  );
 
   // SECURITY: iptables init container - sets up NAT rules in a separate container
   // that shares the agent's network namespace but NEVER gives NET_ADMIN to the agent.
@@ -225,7 +241,7 @@ export function buildIptablesInitService(params: IptablesInitServiceParams): any
     network_mode: 'service:agent',
     // Only mount the init signal volume and the iptables setup script
     volumes: [
-      `${initSignalDir}:/tmp/awf-init:rw`,
+      initSignalMount,
     ],
     environment: {
       // Pass through environment variables needed by setup-iptables.sh
diff --git a/src/services/agent-volumes.ts b/src/services/agent-volumes.ts
@@ -4,6 +4,19 @@ import execa from 'execa';
 import { SslConfig } from '../host-env';
 import { logger } from '../logger';
 import { WrapperConfig } from '../types';
+import {
+  normalizeDockerHostPathPrefix,
+  translateBindMountHostPath,
+  applyHostPathPrefixToVolumes,
+} from './host-path-prefix';
+
+// Re-export for backwards compatibility — call sites that previously imported
+// these helpers from agent-volumes.ts continue to work.
+export {
+  normalizeDockerHostPathPrefix,
+  translateBindMountHostPath,
+  applyHostPathPrefixToVolumes,
+};
 
 // ─── Agent Volumes ────────────────────────────────────────────────────────────
 
@@ -18,47 +31,6 @@ interface AgentVolumesParams {
   initSignalDir: string;
 }
 
-function normalizeDockerHostPathPrefix(prefix: string): string {
-  const trimmed = prefix.trim();
-  if (!trimmed) return '';
-  const withLeadingSlash = trimmed.startsWith('/') ? trimmed : `/${trimmed}`;
-  const withoutTrailingSlash = withLeadingSlash.replace(/\/+$/, '');
-  return withoutTrailingSlash || '/';
-}
-
-function translateBindMountHostPath(mount: string, dockerHostPathPrefix: string): string {
-  const parts = mount.split(':');
-  if (parts.length < 2 || parts.length > 3) {
-    return mount;
-  }
-
-  const [hostPath, containerPath, mode] = parts;
-  if (!hostPath.startsWith('/')) {
-    return mount;
-  }
-
-  // Skip kernel virtual filesystems — /dev, /sys, and /proc are provided by the
-  // Docker daemon's own kernel, not staged runner paths. Prefixing them would look
-  // for non-existent directories under the runner root.
-  // SECURITY: /dev/null must be preserved for credential-hiding overlays.
-  // /proc is not bind-mounted (it's a fresh procfs via mount -t proc in entrypoint.sh
-  // with hidepid=2), but is included defensively to prevent accidental exposure of
-  // /proc/*/environ which contains auth credentials.
-  if (hostPath === '/dev/null' || hostPath.startsWith('/dev') || hostPath.startsWith('/sys') || hostPath.startsWith('/proc')) {
-    return mount;
-  }
-
-  if (dockerHostPathPrefix === '/') {
-    return mount;
-  }
-
-  const translatedHostPath = hostPath === '/'
-    ? dockerHostPathPrefix
-    : `${dockerHostPathPrefix}${hostPath}`;
-
-  return mode ? `${translatedHostPath}:${containerPath}:${mode}` : `${translatedHostPath}:${containerPath}`;
-}
-
 const DEFAULT_DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 
 function resolveDockerSocketPath(config: WrapperConfig): string {
@@ -491,10 +463,7 @@ export function buildAgentVolumes(params: AgentVolumesParams): string[] {
   logger.debug(`Hidden ${chrootCredentialFiles.length} credential file(s) at /host paths`);
 
   if (config.dockerHostPathPrefix) {
-    const dockerHostPathPrefix = normalizeDockerHostPathPrefix(config.dockerHostPathPrefix);
-    if (dockerHostPathPrefix) {
-      return agentVolumes.map(mount => translateBindMountHostPath(mount, dockerHostPathPrefix));
-    }
+    return applyHostPathPrefixToVolumes(agentVolumes, config.dockerHostPathPrefix);
   }
 
   return agentVolumes;
diff --git a/src/services/api-proxy-service.ts b/src/services/api-proxy-service.ts
@@ -11,6 +11,7 @@ import { WrapperConfig, API_PROXY_PORTS, API_PROXY_HEALTH_PORT } from '../types'
 import { pickEnvVars } from '../env-utils';
 import { COPILOT_PLACEHOLDER_TOKEN } from '../constants/placeholders';
 import { NetworkConfig, ImageBuildConfig } from './squid-service';
+import { applyHostPathPrefixToVolumes } from './host-path-prefix';
 
 interface ApiProxyBuildResult {
   /** The api-proxy service definition to add to Docker Compose services. */
@@ -70,10 +71,13 @@ export function buildApiProxyService(params: ApiProxyServiceParams): ApiProxyBui
         ipv4_address: networkConfig.proxyIp,
       },
     },
-    volumes: [
-      // Mount log directory for api-proxy logs
-      `${apiProxyLogsPath}:/var/log/api-proxy:rw`,
-    ],
+    volumes: applyHostPathPrefixToVolumes(
+      [
+        // Mount log directory for api-proxy logs
+        `${apiProxyLogsPath}:/var/log/api-proxy:rw`,
+      ],
+      config.dockerHostPathPrefix,
+    ),
     environment: {
       // Pass API keys securely to sidecar (not visible to agent)
       ...(config.openaiApiKey && { OPENAI_API_KEY: config.openaiApiKey }),
diff --git a/src/services/cli-proxy-service.ts b/src/services/cli-proxy-service.ts
@@ -4,6 +4,7 @@ import { buildRuntimeImageRef } from '../image-tag';
 import { logger } from '../logger';
 import { WrapperConfig, CLI_PROXY_PORT } from '../types';
 import { NetworkConfig, ImageBuildConfig } from './squid-service';
+import { applyHostPathPrefixToVolumes } from './host-path-prefix';
 
 interface CliProxyBuildResult {
   /** The cli-proxy service definition to add to Docker Compose services. */
@@ -52,12 +53,15 @@ export function buildCliProxyService(params: CliProxyServiceParams): CliProxyBui
     },
     // Enable host.docker.internal resolution for connecting to host DIFC proxy
     extra_hosts: ['host.docker.internal:host-gateway'],
-    volumes: [
-      // Log directory for HTTP server logs
-      `${cliProxyLogsPath}:/var/log/cli-proxy:rw`,
-      // Mount host CA cert for TLS verification
-      ...(config.difcProxyCaCert ? [`${config.difcProxyCaCert}:/tmp/proxy-tls/ca.crt:ro`] : []),
-    ],
+    volumes: applyHostPathPrefixToVolumes(
+      [
+        // Log directory for HTTP server logs
+        `${cliProxyLogsPath}:/var/log/cli-proxy:rw`,
+        // Mount host CA cert for TLS verification
+        ...(config.difcProxyCaCert ? [`${config.difcProxyCaCert}:/tmp/proxy-tls/ca.crt:ro`] : []),
+      ],
+      config.dockerHostPathPrefix,
+    ),
     environment: {
       // External DIFC proxy connection info for tcp-tunnel.js
       AWF_DIFC_PROXY_HOST: difcProxyHost,
diff --git a/src/services/host-path-prefix.ts b/src/services/host-path-prefix.ts
@@ -0,0 +1,65 @@
+// Helpers for rewriting Docker bind-mount source paths so the daemon can
+// resolve them on split runner/Docker daemon filesystems (e.g. ARC + DinD).
+//
+// When the runner process and the Docker daemon do not share the same root
+// filesystem, bind-mount sources resolved on the runner side are not visible
+// to the daemon. The user can stage the runner filesystem (or part of it)
+// under a known location inside the daemon (commonly /host) and pass
+// `--docker-host-path-prefix /host` so AWF rewrites every bind-mount source
+// from `/foo` to `/host/foo` before handing the compose file to docker.
+//
+// These helpers are shared by all service builders (agent, iptables-init,
+// squid, api-proxy, cli-proxy) so the rewrite is symmetric across services
+// that share daemon-side directories.
+
+export function normalizeDockerHostPathPrefix(prefix: string): string {
+  const trimmed = prefix.trim();
+  if (!trimmed) return '';
+  const withLeadingSlash = trimmed.startsWith('/') ? trimmed : `/${trimmed}`;
+  const withoutTrailingSlash = withLeadingSlash.replace(/\/+$/, '');
+  return withoutTrailingSlash || '/';
+}
+
+export function translateBindMountHostPath(mount: string, dockerHostPathPrefix: string): string {
+  const parts = mount.split(':');
+  if (parts.length < 2 || parts.length > 3) {
+    return mount;
+  }
+
+  const [hostPath, containerPath, mode] = parts;
+  if (!hostPath.startsWith('/')) {
+    return mount;
+  }
+
+  // Skip kernel virtual filesystems — /dev, /sys, and /proc are provided by the
+  // Docker daemon's own kernel, not staged runner paths. Prefixing them would look
+  // for non-existent directories under the runner root.
+  // SECURITY: /dev/null must be preserved for credential-hiding overlays.
+  // /proc is not bind-mounted (it's a fresh procfs via mount -t proc in entrypoint.sh
+  // with hidepid=2), but is included defensively to prevent accidental exposure of
+  // /proc/*/environ which contains auth credentials.
+  if (hostPath === '/dev/null' || hostPath.startsWith('/dev') || hostPath.startsWith('/sys') || hostPath.startsWith('/proc')) {
+    return mount;
+  }
+
+  if (dockerHostPathPrefix === '/') {
+    return mount;
+  }
+
+  const translatedHostPath = hostPath === '/'
+    ? dockerHostPathPrefix
+    : `${dockerHostPathPrefix}${hostPath}`;
+
+  return mode ? `${translatedHostPath}:${containerPath}:${mode}` : `${translatedHostPath}:${containerPath}`;
+}
+
+// Applies dockerHostPathPrefix translation to every bind mount in the list.
+// Returns the input unchanged when no prefix is set or the prefix normalises
+// to an empty string. Service builders call this at the end of their volume
+// list construction so the rewrite is consistent across the compose stack.
+export function applyHostPathPrefixToVolumes(volumes: string[], dockerHostPathPrefix: string | undefined): string[] {
+  if (!dockerHostPathPrefix) return volumes;
+  const normalized = normalizeDockerHostPathPrefix(dockerHostPathPrefix);
+  if (!normalized) return volumes;
+  return volumes.map(mount => translateBindMountHostPath(mount, normalized));
+}
diff --git a/src/services/squid-service.ts b/src/services/squid-service.ts
@@ -3,6 +3,7 @@ import { SslConfig, SQUID_PORT, SQUID_CONTAINER_NAME } from '../host-env';
 import { parseImageTag, buildRuntimeImageRef } from '../image-tag';
 import { logger } from '../logger';
 import { WrapperConfig } from '../types';
+import { applyHostPathPrefixToVolumes } from './host-path-prefix';
 
 /** Network configuration passed to service builders */
 export interface NetworkConfig {
@@ -56,6 +57,10 @@ export function buildSquidService(params: SquidServiceParams): any {
     squidVolumes.push(`${sslConfig.sslDbPath}:/var/spool/squid_ssl_db:rw`);
   }
 
+  // Apply --docker-host-path-prefix to all bind-mount sources so the daemon
+  // can resolve them on split runner/Docker daemon filesystems (e.g. ARC + DinD).
+  const translatedSquidVolumes = applyHostPathPrefixToVolumes(squidVolumes, config.dockerHostPathPrefix);
+
   // Squid service configuration
   const squidService: any = {
     container_name: SQUID_CONTAINER_NAME,
@@ -64,7 +69,7 @@ export function buildSquidService(params: SquidServiceParams): any {
         ipv4_address: networkConfig.squidIp,
       },
     },
-    volumes: squidVolumes,
+    volumes: translatedSquidVolumes,
     healthcheck: {
       test: ['CMD', 'nc', '-z', 'localhost', '3128'],
       interval: '1s',
