split arm64 build: Docker on Linux, QEMU snapshot on macOS

BilalG1 · BilalG1 · commit 49a20ed01966 · 2026-04-15T18:36:50.000-07:00
Docker is difficult to run on macOS CI runners (colima VZ and QEMU
backends both crash). Split into two stages:
  1. docker-build (Linux): builds arm64 Docker image, exports tarball
  2. qemu-snapshot (macOS): provisions QEMU VM under HVF, captures snapshot

Add SKIP_DOCKER_BUILD=1 to build-image.sh to reuse a pre-built bundle.
diff --git a/.github/workflows/qemu-emulator-build-arm64.yaml b/.github/workflows/qemu-emulator-build-arm64.yaml
@@ -1,9 +1,11 @@
 name: Build QEMU Emulator Image (arm64 / macOS)
 
-# arm64 emulator images are built on a macOS Apple Silicon runner so the
-# snapshot is captured under HVF — the same accelerator developer Macs use.
-# KVM snapshots (from Linux runners) are NOT resumable under HVF because
-# `-cpu max` expands to different feature sets under each accelerator.
+# arm64 emulator images are built in two stages:
+#   1. docker-build (Linux): builds the Docker container image for arm64 and
+#      exports a tarball — Docker is painful to run on macOS CI runners.
+#   2. qemu-snapshot (macOS): boots the image under HVF on Apple Silicon,
+#      provisions it, and captures a snapshot. HVF snapshots are portable to
+#      developer Macs; KVM snapshots are NOT (differing -cpu max features).
 
 on:
   push:
@@ -22,14 +24,68 @@ concurrency:
 
 env:
   EMULATOR_IMAGE_NAME: stack-local-emulator
-  EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images
-  EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run
 
 jobs:
-  build:
-    name: Build QEMU Image (arm64)
+  # ---------- Stage 1: build Docker image on Linux ----------
+  docker-build:
+    name: Build Docker Image (arm64)
+    runs-on: ubicloud-standard-8
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up QEMU user-mode emulation
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - name: Generate emulator env
+        run: node docker/local-emulator/generate-env-development.mjs
+
+      - name: Build arm64 Docker image
+        run: |
+          docker buildx build \
+            --platform linux/arm64 \
+            --tag "$EMULATOR_IMAGE_NAME" \
+            --load \
+            -f docker/local-emulator/Dockerfile \
+            .
+
+      - name: Export Docker image bundle
+        run: |
+          mkdir -p /tmp/bundle
+          docker save "$EMULATOR_IMAGE_NAME" | gzip -c > /tmp/bundle/emulator-arm64-docker-images.tar.gz
+          docker image inspect --format '{{.ID}}' "$EMULATOR_IMAGE_NAME" > /tmp/bundle/emulator-arm64-docker-images.tar.gz.image-ids
+          ls -lh /tmp/bundle/
+
+      - name: Upload Docker bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: arm64-docker-bundle
+          path: /tmp/bundle/
+          retention-days: 1
+          compression-level: 0
+
+  # ---------- Stage 2: QEMU provision + snapshot on macOS (HVF) ----------
+  qemu-snapshot:
+    name: QEMU Snapshot (arm64 / HVF)
+    needs: docker-build
     runs-on: macos-15
     timeout-minutes: 120
+    env:
+      EMULATOR_IMAGE_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/images
+      EMULATOR_RUN_DIR: ${{ github.workspace }}/docker/local-emulator/qemu/run
 
     steps:
       - uses: actions/checkout@v6
@@ -46,17 +102,6 @@ jobs:
       - name: Install system dependencies
         run: brew install qemu socat zstd
 
-      - name: Set up Docker via colima
-        run: |
-          brew install docker docker-buildx colima
-          # Wire up buildx as a CLI plugin
-          mkdir -p ~/.docker
-          echo '{"cliPluginsExtraDirs":["/opt/homebrew/lib/docker/cli-plugins"]}' > ~/.docker/config.json
-          # VZ driver doesn't work on GHA macOS runners — use QEMU backend
-          colima start --vm-type=qemu --cpu 4 --memory 6 --disk 60 --arch aarch64
-          docker info
-          docker buildx version
-
       - name: Verify QEMU + HVF
         run: |
           qemu-system-aarch64 --version
@@ -67,17 +112,26 @@ jobs:
             exit 1
           fi
 
-      - name: Build QEMU image
+      - name: Download Docker bundle
+        uses: actions/download-artifact@v4
+        with:
+          name: arm64-docker-bundle
+          path: ${{ env.EMULATOR_IMAGE_DIR }}/
+
+      - name: Generate emulator env
+        run: node docker/local-emulator/generate-env-development.mjs
+
+      - name: Build QEMU image (provision + snapshot)
         run: |
           chmod +x docker/local-emulator/qemu/build-image.sh
+          # SKIP_DOCKER_BUILD=1 tells build-image.sh to skip the Docker
+          # build + export steps — we already have the bundle from stage 1.
           EMULATOR_PROVISION_TIMEOUT=6000 \
+          SKIP_DOCKER_BUILD=1 \
             docker/local-emulator/qemu/build-image.sh arm64
 
-      - name: Generate emulator env
-        run: node docker/local-emulator/generate-env-development.mjs
-
-      # HVF gives us native-speed arm64 — we can verify the image boots
-      # and services come up, unlike the old cross-arch TCG path.
+      # HVF gives us native-speed arm64 — verify the image boots and
+      # services come up (previously impossible under cross-arch TCG).
       - name: Build stack-cli
         run: |
           pnpm install --frozen-lockfile --filter '@stackframe/stack-cli...'
@@ -87,34 +141,27 @@ jobs:
         env:
           EMULATOR_ARCH: arm64
           EMULATOR_READY_TIMEOUT: 3200
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
         run: node packages/stack-cli/dist/index.js emulator start
 
       - name: Verify services are healthy
         env:
           EMULATOR_ARCH: arm64
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
         run: node packages/stack-cli/dist/index.js emulator status
 
       - name: Stop emulator
         if: always()
         env:
           EMULATOR_ARCH: arm64
-          EMULATOR_IMAGE_DIR: ${{ env.EMULATOR_IMAGE_DIR }}
-          EMULATOR_RUN_DIR: ${{ env.EMULATOR_RUN_DIR }}
         run: node packages/stack-cli/dist/index.js emulator stop
 
       - name: Print serial log on failure
         if: failure()
-        run: |
-          tail -100 "$EMULATOR_RUN_DIR/vm/serial.log" 2>/dev/null || true
+        run: tail -100 "$EMULATOR_RUN_DIR/vm/serial.log" 2>/dev/null || true
 
       - name: Package image
         run: |
-          BASE_IMG="docker/local-emulator/qemu/images/stack-emulator-arm64.qcow2"
-          SAVEVM="docker/local-emulator/qemu/images/stack-emulator-arm64.savevm.zst"
+          BASE_IMG="$EMULATOR_IMAGE_DIR/stack-emulator-arm64.qcow2"
+          SAVEVM="$EMULATOR_IMAGE_DIR/stack-emulator-arm64.savevm.zst"
           cp "$BASE_IMG" "stack-emulator-arm64.qcow2"
           if [ -f "$SAVEVM" ]; then
             cp "$SAVEVM" "stack-emulator-arm64.savevm.zst"
diff --git a/docker/local-emulator/qemu/build-image.sh b/docker/local-emulator/qemu/build-image.sh
@@ -657,8 +657,12 @@ BUILD_ENV_FILE="$REPO_ROOT/docker/local-emulator/.env.development"
 for arch in "${TARGET_ARCHS[@]}"; do
   local_base="$IMAGE_DIR/debian-${DEBIAN_VERSION}-base-${arch}.qcow2"
   download_cloud_image "$arch" "$local_base"
-  build_local_emulator_image "$arch"
-  prepare_bundle_artifacts "$arch"
+  if [ "${SKIP_DOCKER_BUILD:-0}" = "1" ]; then
+    log "SKIP_DOCKER_BUILD=1: reusing pre-built Docker bundle"
+  else
+    build_local_emulator_image "$arch"
+    prepare_bundle_artifacts "$arch"
+  fi
   build_one "$arch"
 done
 
