Skip to content

Commit a8e28fe

Browse files
committed
fix: resolve Docker build timeouts on Modal runner with tmpfs, pre-pulled images, and BuildKit
Three infrastructure fixes to address slow Docker builds inside Modal Sandboxes caused by gVisor networking and filesystem overhead: - Add tmpfs mount for /var/lib/docker in start-dockerd.sh (gVisor requires tmpfs for overlay2 storage driver; without it Docker falls back to vfs which is 10-50x slower) - Pre-pull common base images (node:22-alpine, python:3.12-alpine, alpine:3.19) into runner image using Modal's optimized image system, eliminating 140s+ runtime pulls through gVisor netstack - Export DOCKER_BUILDKIT=1 in sandbox startup to ensure BuildKit is always enabled for Docker builds
1 parent 8662fff commit a8e28fe

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

app.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,11 @@ def _get_gpu_config(gpu_key: str):
173173
update-alternatives --set iptables /usr/sbin/iptables-legacy 2>/dev/null || true
174174
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy 2>/dev/null || true
175175
176+
# gVisor requires tmpfs for Docker overlay2 storage driver.
177+
# Without this, Docker falls back to the vfs driver (10-50x slower).
178+
mkdir -p /var/lib/docker
179+
mount -t tmpfs -o size=4G tmpfs /var/lib/docker
180+
176181
exec /usr/bin/dockerd --iptables=false --ip6tables=false > /var/log/dockerd.log 2>&1
177182
"""
178183

@@ -214,6 +219,17 @@ def _get_gpu_config(gpu_key: str):
214219
"update-alternatives --set iptables /usr/sbin/iptables-legacy",
215220
"update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy",
216221
)
222+
# Layer 1b: Pre-pull common base images using Modal's optimized image system
223+
# This avoids 140s+ Docker pulls through gVisor's slow netstack at runtime.
224+
.add_local_file(_TEMP_DOCKERD_PATH, "/start-dockerd.sh", copy=True)
225+
.run_commands("chmod +x /start-dockerd.sh")
226+
.run_commands(
227+
"bash /start-dockerd.sh & sleep 5 && until docker info >/dev/null 2>&1; do sleep 1; done",
228+
"docker pull node:22-alpine",
229+
"docker pull python:3.12-alpine",
230+
"docker pull alpine:3.19",
231+
"kill %1 2>/dev/null || true",
232+
)
217233
# Layer 2: Python deps (semi-stable)
218234
.apt_install("python3", "python3-pip", "python-is-python3")
219235
.pip_install("fastapi==0.115.0", "httpx==0.27.0")
@@ -223,9 +239,6 @@ def _get_gpu_config(gpu_key: str):
223239
f"curl -L https://github.com/actions/runner/releases/download/v{RUNNER_VERSION}/actions-runner-linux-x64-{RUNNER_VERSION}.tar.gz | tar -xz -C /actions-runner", # noqa: E501
224240
"/actions-runner/bin/installdependencies.sh",
225241
)
226-
# Layer 4: Local files (most volatile)
227-
.add_local_file(_TEMP_DOCKERD_PATH, "/start-dockerd.sh", copy=True)
228-
.run_commands("chmod +x /start-dockerd.sh")
229242
)
230243

231244
app = modal.App("modal-github-runner")
@@ -733,6 +746,7 @@ async def github_webhook(request: Request):
733746
" && until docker info > /dev/null 2>&1; do sleep 1; done"
734747
" && cd /actions-runner"
735748
" && export RUNNER_ALLOW_RUNASROOT=1"
749+
" && export DOCKER_BUILDKIT=1"
736750
" && exec ./run.sh --jitconfig $GHA_JIT_CONFIG"
737751
)
738752

0 commit comments

Comments
 (0)