refactor: switch devctr build to be multi-staged

JamesC1305 · JamesC1305 · commit 3b175bcb47d9 · 2026-04-08T15:50:34.000+01:00
Switch from a single stage build to a multi-staged build for the devctr.
Although we don't often rebuild this, this should reduce the time taken
when we do, due to unchanged layers now being cached.

In my testing, this cut the full build time from ~1300s to ~600s without
inflating the image size further. We also don't have to manually clean
up after earlier build stages, as they are not included in the final
image.

Stage dependency graph:

                                   +--- qemu-builder -------+
                                   |                        |
                                   +--- libseccomp-builder -+
                                   |                        |
   base-image +--------------------+--- iperf3-builder -----+--- devctr
              |                    |                        |
              |                    +--- git-secrets-builder +
              |                                             |
              +--- apt-base                                 |
                      |                                     |
                      |                                     |
                  python-deps      +--- crosvm-builder -----+
                      |            |                        |
                 rust-toolchain ---+------------------------+

 Parallel builders (from base-image):
   qemu-builder, libseccomp-builder, iperf3-builder, git-secrets-builder

 Sequential base chain:
   base-image -&gt; apt-base -&gt; python-deps -&gt; rust-toolchain

 Rust fork (from rust-toolchain):
   crosvm-builder  (build deps thrown away, only binary copied)
   devctr           (cargo tools, kani, nightly, then COPY
                      from all builders)

Signed-off-by: James Curtis &lt;jxcurtis@amazon.co.uk&gt;
diff --git a/tools/devctr/Dockerfile b/tools/devctr/Dockerfile
@@ -1,26 +1,56 @@
-FROM public.ecr.aws/lts/ubuntu:24.04
-
-# TODO: use a multi-stage build to reduce the download size when updating this container.
-# The Rust toolchain layer will get updated most frequently, but we could keep the system
-# dependencies layer intact for much longer.
+# Building the devctr is a multi-stage process. The dependency graph is as follows:
+#
+# Stage dependency graph:
+#
+#                                   +--- qemu-builder -------+
+#                                   |                        |
+#                                   +--- libseccomp-builder -+
+#                                   |                        |
+#   base-image +--------------------+--- iperf3-builder -----+--- devctr
+#              |                    |                        |
+#              |                    +--- git-secrets-builder +
+#              |                                             |
+#              +--- apt-base                                 |
+#                      |                                     |
+#                      |                                     |
+#                  python-deps      +--- crosvm-builder -----+
+#                      |            |                        |
+#                 rust-toolchain ---+------------------------+
+#
+# Parallel builders (from base-image):
+#   qemu-builder, libseccomp-builder, iperf3-builder, git-secrets-builder
+#
+# Sequential base chain:
+#   base-image -> apt-base -> python-deps -> rust-toolchain
+#
+# Rust fork (from rust-toolchain):
+#   crosvm-builder  (build deps thrown away, only binary copied)
+#   devctr           (cargo tools, kani, nightly, then COPY from all builders)
+#
 
-ARG RUST_TOOLCHAIN="1.94.1"
-ARG TMP_BUILD_DIR=/tmp/build
+# ============================================================
+# Base image: Shared between builders
+# ============================================================
+FROM public.ecr.aws/lts/ubuntu:24.04 AS base-image
 ARG DEBIAN_FRONTEND=noninteractive
-ARG PIP_BREAK_SYSTEM_PACKAGES=1
 ARG ARCH
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends \
+        git gcc make libc-dev ca-certificates musl-tools
 
-ENV CARGO_HOME=/usr/local/rust
-ENV RUSTUP_HOME=/usr/local/rust
-ENV PATH="$PATH:$CARGO_HOME/bin"
-ENV LC_ALL=C.UTF-8
+# help musl-gcc find linux headers
+RUN cd /usr/include/$ARCH-linux-musl \
+    && ln -s ../$ARCH-linux-gnu/asm asm \
+    && ln -s ../linux linux \
+    && ln -s ../asm-generic asm-generic
+
+# ============================================================
+# Builder: Qemu vhost-user-blk backend
+# ============================================================
+FROM base-image AS qemu-builder
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
 ENV QEMU_VER="8.1.1"
-ENV CROSVM_VER="9d542e6dafa3a85acd1fb6cd6f1adfa1331c4e96"
-ENV CROSVM_TOOLCHAIN_VER="1.68.2"
-ENV LIBSECCOMP_VER="v2.6.0"
 
-# Build and install Qemu vhost-user-blk backend
-#
 RUN apt-get update \
     && apt-get -y install --no-install-recommends \
         curl gpg gpg-agent \
@@ -35,27 +65,65 @@ RUN apt-get update \
     && tar xf qemu-${QEMU_VER}.tar.xz && cd qemu-${QEMU_VER} \
     && ./configure && make -j $(nproc) contrib/vhost-user-blk/vhost-user-blk \
     && strip ./build/contrib/vhost-user-blk/vhost-user-blk \
-    && cp -a ./build/contrib/vhost-user-blk/vhost-user-blk /usr/local/bin \
-    && pip3 uninstall -y meson \
-    && apt-get purge -y \
-        curl gpg gpg-agent \
-        build-essential ninja-build libglib2.0-dev libpixman-1-dev flex bison \
-    && apt-get autoremove -y \
-    && cd && rm -r /tmp/qemu_build
+    && cp -a ./build/contrib/vhost-user-blk/vhost-user-blk /usr/local/bin
+
+# ============================================================
+# Builder: static libseccomp (musl)
+# ============================================================
+FROM base-image AS libseccomp-builder
+ENV LIBSECCOMP_VER="v2.6.0"
+
+# Install static version of libseccomp
+# We need to compile from source because
+# libseccomp provided by the distribution is not
+# compiled with musl-gcc and we need this
+# for our musl builds.
+# We specify the tag in order to have a fixed version
+# of the library.
+RUN apt-get -y install --no-install-recommends \
+        libtool gperf autoconf automake \
+    && git clone https://github.com/seccomp/libseccomp /tmp/libseccomp \
+    && cd /tmp/libseccomp \
+    && git checkout tags/${LIBSECCOMP_VER} \
+    && ./autogen.sh \
+    && CC="musl-gcc -static" ./configure --enable-static=yes --enable-shared=false \
+    && make DESTDIR=/staging install
+
+# ============================================================
+# Builder: iperf3-vsock
+# ============================================================
+FROM base-image AS iperf3-builder
+RUN git clone https://github.com/stefano-garzarella/iperf-vsock /tmp/iperf-vsock \
+    && cd /tmp/iperf-vsock && git checkout 9245f9a \
+    && mkdir build && cd build \
+    && ../configure "LDFLAGS=--static" --disable-shared && make \
+    && cp src/iperf3 /usr/local/bin/iperf3-vsock
+
+# ============================================================
+# Builder: git-secrets
+# ============================================================
+FROM base-image AS git-secrets-builder
+RUN git clone https://github.com/awslabs/git-secrets /tmp/git-secrets \
+    && cd /tmp/git-secrets \
+    && make DESTDIR=/staging install
+
+# ============================================================
+# Base: apt packages (changes very rarely)
+# ============================================================
+FROM base-image AS apt-base
+ARG ARCH
+ENV LC_ALL=C.UTF-8
 
-# Install system dependencies
-#
 RUN apt-get update \
     && apt-get -y install --no-install-recommends \
         # essential build tools
-        gcc make libc-dev binutils-dev libssl-dev \
+        binutils-dev libssl-dev \
         # Useful utilities
         gdbserver \
         # Needed in order to be able to compile `userfaultfd-sys`.
         clang \
         curl \
         file \
-        git \
         jq \
         less \
         libbfd-dev \
@@ -80,18 +148,44 @@ RUN apt-get update \
         python3-seccomp \
         # for aws-lc-rs
         cmake \
-        # for Qemu vhost-user-blk backend
-        libglib2.0-dev \
         # for crosvm (vhost-user-blk backend)
         libcap2 \
         # for debugging
         gdb strace trace-cmd \
     && rm -rf /var/lib/apt/lists/*
 
+
+# Add cross-compile toolchain for devtool checkbuild command (rust target added in devctr stage)
+RUN case "${ARCH}" in \
+    "aarch64") \
+        apt-get update && apt install -y gcc-x86-64-linux-gnu libc6-dev-amd64-cross linux-libc-dev-amd64-cross \
+        ;; \
+    "x86_64") \
+        apt-get update && apt install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross linux-libc-dev-arm64-cross \
+        ;; \
+    *) echo "Unsupported arch ${ARCH}" && exit 1 ;; \
+    esac \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Download codecov uploader
+RUN cd /usr/local/bin \
+    && (if [ "$ARCH" = "x86_64" ]; then  \
+      curl -O https://uploader.codecov.io/latest/linux/codecov; else \
+      curl -O https://uploader.codecov.io/latest/aarch64/codecov; fi) \
+    && chmod +x codecov
+
+# ============================================================
+# Base: Python/Poetry deps (changes with poetry.lock)
+# ============================================================
+FROM apt-base AS python-deps
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
+ARG ARCH
+ARG VENV="/opt/venv"
+
 RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="/root/.local/bin:$PATH"
 
-ARG VENV="/opt/venv"
 COPY tools/devctr/poetry.lock /tmp/poetry/
 COPY tools/devctr/pyproject.toml /tmp/poetry/
 RUN cd /tmp/poetry \
@@ -106,24 +200,50 @@ ENV PATH=$VENV/bin:$PATH
 # apt-get installs it globally, to manually copy it into the venv
 RUN cp /usr/lib/python3/dist-packages/seccomp.cpython-312-"$ARCH"-linux-gnu.so "$VENV"/lib/python3.12/site-packages/
 
-RUN git clone https://github.com/awslabs/git-secrets /tmp/git-secrets \
-    && cd /tmp/git-secrets \
-    && make install \
-    && cd - \
-    && rm -rf /tmp/git-secrets
+# ============================================================
+# Base: Rust toolchain (current version)
+# ============================================================
+FROM python-deps AS rust-toolchain
+ARG RUST_TOOLCHAIN="1.94.1"
+ENV CARGO_HOME=/usr/local/rust
+ENV RUSTUP_HOME=/usr/local/rust
+ENV PATH="$PATH:$CARGO_HOME/bin"
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
+    && rustup target add x86_64-unknown-linux-musl \
+    && rustup target add aarch64-unknown-linux-musl \
+    && rustup component add llvm-tools-preview clippy rustfmt
+
+# ============================================================
+# Builder: crosvm (used as vhost-user-blk backend)
+# ============================================================
+FROM rust-toolchain AS crosvm-builder
+ENV CROSVM_VER="9d542e6dafa3a85acd1fb6cd6f1adfa1331c4e96"
+RUN apt-get update && \
+    apt-get -y install --no-install-recommends \
+        libcap2 \
+        libcap-dev \
+        protobuf-compiler \
+    && git clone https://github.com/google/crosvm.git /tmp/crosvm \
+    && cd /tmp/crosvm && git checkout ${CROSVM_VER} \
+    && git submodule update --init \
+    && cargo build --no-default-features --release \
+    && strip ./target/release/crosvm \
+    && cp -a ./target/release/crosvm /usr/local/bin
 
-# Running the three as a single dockerfile command to avoid inflation of the image:
-# - Install the Rust toolchain.
+# ============================================================
+# Devctr (final): Kani, cargo targets and assembled artifacts
+# ============================================================
+FROM rust-toolchain AS devctr
+ARG ARCH
+ENV CARGO_HOME=/usr/local/rust
+
+# Run these as a single dockerfile command to avoid inflation of the image:
+# - Install required binary crates, including cargo-afl (fuzzer) and Kani (formal verification)
 # - Kani always installs _some_ nightly toolchain, we reuse it for the seccomp filter analysis test. Dynamically
 #   determine the exact toolchain name, and install more components into it.
-# - Build and install crosvm (used as vhost-user-blk backend)
-# - Clean up cargo compilation directories
 # - Always install both x86_64 and aarch64 musl targets, as our rust-toolchain.toml would force on-the-fly installation of both anyway
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain "$RUST_TOOLCHAIN" \
-    && rustup target add x86_64-unknown-linux-musl \
-    && rustup target add aarch64-unknown-linux-musl \
-    && rustup component add llvm-tools-preview clippy rustfmt \
-    && cargo install --locked grcov cargo-sort cargo-afl \
+# - Clean up cargo compilation directories
+RUN cargo install --locked grcov cargo-sort cargo-afl \
     && cargo install --locked cargo-deny --version 0.19.0 \
     && cargo install --locked kani-verifier --version 0.67.0 && cargo kani setup \
     \
@@ -132,88 +252,20 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-too
     && rustup target add "$ARCH"-unknown-linux-musl --toolchain "$NIGHTLY_TOOLCHAIN" \
     && cargo +"$NIGHTLY_TOOLCHAIN" install cargo-udeps \
     \
-    && apt-get update \
-    && apt-get -y install --no-install-recommends \
-        libcap-dev \
-        protobuf-compiler \
-    && git clone https://github.com/google/crosvm.git /tmp/crosvm \
-    && cd /tmp/crosvm && git checkout ${CROSVM_VER} \
-    && git submodule update --init \
-    && cargo build --no-default-features --release \
-    && strip ./target/release/crosvm \
-    && cp -a ./target/release/crosvm /usr/local/bin \
-    && apt-get purge -y \
-        libcap-dev \
-        protobuf-compiler \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && rustup toolchain uninstall ${CROSVM_TOOLCHAIN_VER}-${ARCH}-unknown-linux-gnu \
-    && cd && rm -r /tmp/crosvm \
+    && case "${ARCH}" in \
+        "aarch64") rustup target add x86_64-unknown-linux-gnu ;; \
+        "x86_64")  rustup target add aarch64-unknown-linux-gnu ;; \
+       esac \
     \
     && rm -rf "$CARGO_HOME/registry" \
     && rm -rf "$CARGO_HOME/git"
 
-# help musl-gcc find linux headers
-RUN cd /usr/include/$ARCH-linux-musl \
-    && ln -s ../$ARCH-linux-gnu/asm asm \
-    && ln -s ../linux linux \
-    && ln -s ../asm-generic asm-generic
-
-# Install static version of libseccomp
-# We need to compile from source because
-# libseccomp provided by the distribution is not
-# compiled with musl-gcc and we need this
-# for our musl builds.
-# We specify the tag in order to have a fixed version
-# of the library.
-RUN apt-get update \
-    && apt-get -y install \
-        libtool gperf \
-    && git clone https://github.com/seccomp/libseccomp /tmp/libseccomp \
-    && cd /tmp/libseccomp \
-    && git checkout tags/${LIBSECCOMP_VER} \
-    && ./autogen.sh \
-    && CC="musl-gcc -static" ./configure --enable-static=yes --enable-shared=false \
-    && make install \
-    && cd \
-    && apt-get purge -y \
-        libtool gperf \
-    && apt-get autoremove -y \
-    && rm -rf /tmp/libseccomp
-
-# Build iperf3-vsock
-RUN mkdir "$TMP_BUILD_DIR" && cd "$TMP_BUILD_DIR" \
-    && git clone https://github.com/stefano-garzarella/iperf-vsock \
-    && cd iperf-vsock && git checkout 9245f9a \
-    && mkdir build && cd build \
-    && ../configure "LDFLAGS=--static" --disable-shared && make \
-    && cp src/iperf3 /usr/local/bin/iperf3-vsock \
-    && cd / \
-    && rm -rf "$TMP_BUILD_DIR"
-
-# Download the codecov.io uploader
-RUN cd /usr/local/bin \
-    && (if [ "$ARCH" = "x86_64" ]; then  \
-      curl -O https://uploader.codecov.io/latest/linux/codecov; else \
-      curl -O https://uploader.codecov.io/latest/aarch64/codecov; fi) \
-    && chmod +x codecov \
-    && cd -
-
-# Add cross-compile toolchain for devtool checkbuild command
-RUN case "${ARCH}" in \
-    "aarch64") \
-        apt install -y gcc-x86-64-linux-gnu libc6-dev-amd64-cross linux-libc-dev-amd64-cross && \
-        rustup target add x86_64-unknown-linux-gnu \
-        ;; \
-    "x86_64") \
-        apt install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross linux-libc-dev-arm64-cross && \
-        rustup target add aarch64-unknown-linux-gnu \
-        ;; \
-    *) \
-        echo "Unsupported arch ${ARCH}" && \
-        exit 1 \
-        ;; \
-    esac
+# Copy pre-built artifacts LAST — builder changes don't invalidate Rust layer
+COPY --from=qemu-builder /usr/local/bin/vhost-user-blk /usr/local/bin/
+COPY --from=libseccomp-builder /staging /
+COPY --from=iperf3-builder /usr/local/bin/iperf3-vsock /usr/local/bin/
+COPY --from=git-secrets-builder /staging /
+COPY --from=crosvm-builder /usr/local/bin/crosvm /usr/local/bin/
 
 ADD tools/devctr/ctr_gitconfig /root/.gitconfig
 
