fix(os): rebuild pinned tools for production images

Author: kvinwang

os/mkosi/build.sh

@@ -14,16 +14,23 @@ case "$action" in
   *) echo "Usage: $0 {image|dev-image|repro-check|lint} [build-dir]" >&2; exit 2 ;;
 esac
 if [[ $action == lint ]]; then exec "$SELF/tests/acceptance.sh"; fi
-command -v mkosi >/dev/null || { echo 'mkosi >= 26 is required' >&2; exit 1; }
+command -v mkosi >/dev/null || { echo "mkosi $MKOSI_VERSION is required" >&2; exit 1; }
 actual=$(mkosi --version | awk '{print $2}' | cut -d. -f1)
-(( actual >= MKOSI_MIN_VERSION )) || { echo "mkosi >= $MKOSI_MIN_VERSION required" >&2; exit 1; }
+[[ $actual == "$MKOSI_VERSION" ]] || {
+  echo "mkosi $MKOSI_VERSION required, found $actual" >&2; exit 1;
+}
 export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(git -C "$ROOT" log -1 --format=%ct)}
 export TZ=UTC LC_ALL=C
 
 build_one() {
   local out=$1 work=$2 flavor=$3
   local kstage="$work/kernel-stage" tree="$work/rootfs"
   rm -rf "$work" "$out"; mkdir -p "$out"
+  # Production builds reconstruct the tools tree from the immutable snapshot;
+  # no previously generated build environment is trusted.
+  if [[ $action != dev-image ]]; then
+    mkosi --directory "$SELF" clean -f
+  fi
 
   mkosi_args=(
     --directory "$SELF"

os/mkosi/mkosi.build

@@ -30,6 +30,12 @@ else
 fi
 export PATH="/opt/dstack-toolchains/rust/bin:/opt/dstack-toolchains/go/bin:$PATH"
 "$SELF/scripts/install-toolchains.sh" "$toolchain_downloads"
+printf 'gcc: '; gcc --version | head -1
+printf 'g++: '; g++ --version | head -1
+printf 'ld: '; ld --version | head -1
+printf 'cmake: '; cmake --version | head -1
+dpkg-query -W -f='build-package: ${binary:Package}=${Version}\n' \
+  gcc g++ binutils cmake make ninja-build dwarves
 
 work=/var/tmp/dstack-component-work
 kernel_stage=/var/tmp/dstack-kernel-stage

os/mkosi/scripts/component-framework.sh

@@ -89,7 +89,7 @@ component_run() {
           "${SOURCE_DATE_EPOCH:?}" "$(uname -m)"
         key_file "$definition" "$SELF/scripts/component-framework.sh" \
           "$SELF/scripts/dev-cache.sh" "$SELF/scripts/install-toolchains.sh" \
-          "$SELF/mkosi.build" "$SELF/mkosi.conf"
+          "$SELF/mkosi.build" "$SELF/mkosi.conf" "$SELF/mkosi.tools.conf"
         component_cache_key
     } | sha256sum | cut -d' ' -f1)
     COMPONENT_KEYS[$name]=$key

os/mkosi/tests/acceptance.sh

@@ -61,7 +61,11 @@ grep -q 'DSTACK_COMPONENT_CACHE.*dev-image' "$D/build.sh"
 grep -q 'scripts/build-components.sh' "$D/mkosi.build"
 grep -q '^BuildPackages=' "$D/mkosi.conf"
 grep -q '^ToolsTree=yes$' "$D/mkosi.conf"
+grep -q '^Snapshot=20260721T000000Z$' "$D/mkosi.tools.conf"
+grep -q 'clean -f' "$D/build.sh"
 grep -q 'install-toolchains.sh' "$D/mkosi.build"
+grep -q '^RUST_TOOLCHAIN_VERSION=1.92.0$' "$D/versions.env"
+grep -q '^GO_TOOLCHAIN_VERSION=1.22.2$' "$D/versions.env"
 for component in dstack-rust container-stack sysbox nvattest kernel nvidia zfs ovmf; do
   definition="$D/components/$component/$component.sh"
   grep -q "^COMPONENT_NAME=$component$" "$definition"

os/mkosi/versions.env

@@ -4,7 +4,7 @@ KERNEL_VERSION=7.1.4
 KERNEL_SHA256=1c63922a119675d38e3ae0f8f6ee07f15c41a786ab9ed66563749bb8c9a08e2e
 DEBIAN_RELEASE=trixie
 DEBIAN_SNAPSHOT=20260721T000000Z
-MKOSI_MIN_VERSION=26
+MKOSI_VERSION=26
 RUST_TOOLCHAIN_VERSION=1.92.0
 RUSTC_TOOLCHAIN_SHA256=78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0
 CARGO_TOOLCHAIN_SHA256=e5e12be2c7126a7036c8adf573078a28b92611f5767cc9bd0a6f7c83081df103