fix(os): make mkosi builds reproducible

Author: kvinwang

os/mkosi/build.sh

@@ -7,6 +7,7 @@ ROOT=$(cd "$SELF/../.." && pwd)
 source "$SELF/versions.env"
 action=${1:-image}
 BUILD_DIR=${2:-$SELF/build}
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
 case "$action" in
   image|repro-check|lint) ;;
   *) echo "Usage: $0 {image|repro-check|lint} [build-dir]" >&2; exit 2 ;;
@@ -66,6 +67,9 @@ EOF
   mkosi --directory "$SELF" --force --extra-tree="$stage" "${devargs[@]}" \
     --format=directory --output-directory="$work" --output=rootfs \
     --compress-output=no --bootable=no build
+  # ldconfig's binary auxiliary cache records traversal-dependent ordering.
+  # /var is volatile at runtime, so ship no host-generated cache.
+  rm -f "$tree/var/cache/ldconfig/aux-cache"
   mkdir -p "$tree/usr/lib/modules"
   cp -a "$kstage/usr/lib/modules/." "$tree/usr/lib/modules/"
   "$SELF/tests/check-parity.py" "$SELF/parity.json" "$tree" "$kstage" "$flavor"

os/mkosi/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch

@@ -0,0 +1,15 @@
+# Prefix-map arguments contain the intentionally different clean-build path.
+# They must affect compilation without becoming runtime version metadata.
+diff --git a/Makefile b/Makefile
+index 1a5115c..87c501c 100644
+--- a/Makefile
++++ b/Makefile
+@@ -203,7 +203,7 @@ DEPENDENCIES   := $(BIN_OBJS:%.o=%.d) $(LIB_OBJS:%.lo=%.d)
+ $(BUILD_DEFS):
+ 	@printf '#define BUILD_DATE     "%s"\n' '$(strip $(DATE))' >$(BUILD_DEFS)
+ 	@printf '#define BUILD_COMPILER "%s " __VERSION__\n' '$(notdir $(COMPILER))' >>$(BUILD_DEFS)
+-	@printf '#define BUILD_FLAGS    "%s"\n' '$(strip $(CPPFLAGS) $(CFLAGS) $(LDFLAGS))' >>$(BUILD_DEFS)
++	@printf '#define BUILD_FLAGS    "%s"\n' '$(strip $(filter-out -ffile-prefix-map=% -fmacro-prefix-map=%,$(CPPFLAGS) $(CFLAGS) $(LDFLAGS)))' >>$(BUILD_DEFS)
+ 	@printf '#define BUILD_REVISION "%s"\n' '$(strip $(REVISION))' >>$(BUILD_DEFS)
+ 	@printf '#define BUILD_PLATFORM "%s"\n' '$(strip $(PLATFORM))' >>$(BUILD_DEFS)
+ 

os/mkosi/scripts/build-container-stack.sh

@@ -7,6 +7,8 @@ SELF="$ROOT/os/mkosi"
 source "$SELF/versions.env"
 BUILD_DIR=${1:?build directory required}
 STAGE=${2:?rootfs staging tree required}
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
+STAGE=$(realpath -m "$STAGE")
 mkdir -p "$BUILD_DIR/downloads" "$STAGE/usr/bin"
 
 checkout() {
@@ -17,6 +19,7 @@ checkout() {
   git -C "$dir" fetch -q --depth=1 origin "$rev"
   git -C "$dir" checkout -q --detach FETCH_HEAD
   git -C "$dir" reset -q --hard "$rev"
+  git -C "$dir" clean -qfdx
 }
 fetch_sha256() {
   local url=$1 sha=$2 output=$3
@@ -26,8 +29,17 @@ fetch_sha256() {
 
 lnc="$BUILD_DIR/libnvidia-container"
 checkout https://github.com/NVIDIA/libnvidia-container.git "$LIBNVIDIA_CONTAINER_REVISION" "$lnc"
+patch -d "$lnc" -p1 --fuzz=0 < \
+  "$SELF/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch"
+export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$BUILD_DIR=/usr/src/container-stack -fmacro-prefix-map=$BUILD_DIR=/usr/src/container-stack"
+export LDFLAGS="${LDFLAGS:-} -Wl,--build-id=none"
+export CGO_CFLAGS="$CFLAGS"
+export GOFLAGS="${GOFLAGS:-} -trimpath -buildvcs=false"
 make -C "$lnc" -j"${JOBS:-$(nproc)}" LIB_VERSION=1.18.1 \
   WITH_NVCGO=yes WITH_LIBELF=yes WITH_TIRPC=yes all
+# Upstream adds a CRC of its unshipped split-debug file. The debug file is not
+# installed, and its CRC varies with the clean build directory.
+objcopy --remove-section=.gnu_debuglink "$lnc/libnvidia-container.so.1.18.1"
 install -m0755 "$lnc/nvidia-container-cli" "$STAGE/usr/bin/"
 install -Dm0755 "$lnc/libnvidia-container.so.1.18.1" "$STAGE/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.18.1"
 ln -sfn libnvidia-container.so.1.18.1 "$STAGE/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1"

os/mkosi/scripts/build-kernel.sh

@@ -7,13 +7,37 @@ MKOSI_DIR="$ROOT/os/mkosi"
 source "$MKOSI_DIR/versions.env"
 BUILD_DIR=${1:?build directory required}
 STAGING=${2:?staging tree required}
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
+STAGING=$(realpath -m "$STAGING")
 JOBS=${JOBS:-$(nproc)}
 export KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH required}"
 export KBUILD_BUILD_USER=dstack KBUILD_BUILD_HOST=reproducible
 export KBUILD_BUILD_VERSION=1
 export KCONFIG_NOTIMESTAMP=1
+# Neither the source nor output directory may leak into DWARF/BTF or module
+# metadata: repro-check deliberately builds in two differently named trees.
+kernel_map="-fdebug-prefix-map=$BUILD_DIR=/usr/src/linux -fmacro-prefix-map=$BUILD_DIR=/usr/src/linux"
+export KCFLAGS="${KCFLAGS:-} $kernel_map"
+export KAFLAGS="${KAFLAGS:-} $kernel_map"
+export KCPPFLAGS="${KCPPFLAGS:-} $kernel_map"
 
 mkdir -p "$BUILD_DIR/downloads" "$BUILD_DIR/kernel-build" "$STAGING"
+# pahole 1.25 produces a different BTF type order when its encoder runs with
+# Kbuild's parallel job count. Keep compilation parallel, but serialize BTF
+# encoding so vmlinux and bzImage are byte-for-byte reproducible.
+pahole_wrapper="$BUILD_DIR/pahole-reproducible"
+cat > "$pahole_wrapper" <<'EOF'
+#!/bin/bash
+set -euo pipefail
+args=()
+for arg; do
+    # dwarves 1.25's parallel and optimized-function encoders both depend on
+    # unstable traversal order. Neither is needed for BTF correctness.
+    [[ $arg == -j* || $arg == --btf_gen_optimized ]] || args+=("$arg")
+done
+exec pahole -j1 "${args[@]}"
+EOF
+chmod 0755 "$pahole_wrapper"
 tarball="$BUILD_DIR/downloads/linux-$KERNEL_VERSION.tar.xz"
 if [[ ! -f $tarball ]]; then
     curl --fail --location --retry 3 -o "$tarball.tmp" \
@@ -32,14 +56,14 @@ for patch in \
     patch -d "$src" -p1 --fuzz=0 < "$patch"
 done
 
-make -C "$src" O="$BUILD_DIR/kernel-build" x86_64_defconfig
+make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" x86_64_defconfig
 "$src/scripts/kconfig/merge_config.sh" -m -O "$BUILD_DIR/kernel-build" \
     "$BUILD_DIR/kernel-build/.config" "$MKOSI_DIR/kernel.config"
-make -C "$src" O="$BUILD_DIR/kernel-build" olddefconfig
+make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" olddefconfig
 "$MKOSI_DIR/scripts/check-kernel-config.sh" "$BUILD_DIR/kernel-build/.config"
-make -C "$src" O="$BUILD_DIR/kernel-build" -j"$JOBS" bzImage modules
+make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" -j"$JOBS" bzImage modules
 make -C "$src" O="$BUILD_DIR/kernel-build" \
-    INSTALL_MOD_PATH="$STAGING" modules_install
+    PAHOLE="$pahole_wrapper" INSTALL_MOD_PATH="$STAGING" modules_install
 # Debian is usr-merged: /lib is a symlink. Normalize modules into /usr/lib so
 # ExtraTrees never tries to replace that symlink with a directory.
 if [[ -d $STAGING/lib/modules ]]; then

os/mkosi/scripts/build-nvattest.sh

@@ -7,23 +7,41 @@ SELF="$ROOT/os/mkosi"
 source "$SELF/versions.env"
 BUILD_DIR=${1:?build directory required}
 STAGE=${2:?rootfs staging tree required}
-src="$BUILD_DIR/attestation-sdk"
-build="$BUILD_DIR/build"
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
+STAGE=$(realpath -m "$STAGE")
+checkout="$BUILD_DIR/attestation-sdk"
+# Cargo includes the literal RUSTFLAGS in crate disambiguators. A prefix-map
+# containing work-a/work-b would therefore change Rust symbol hashes even
+# though the mapped paths are equal. Serialize this component in a stable,
+# clean pathname and keep the per-build checkout as the verified input.
+canonical=/var/tmp/dstack-mkosi-nvattest
+exec 9>"$canonical.lock"
+flock 9
+src="$canonical/attestation-sdk"
+build="$canonical/build"
 
-if [[ ! -d $src/.git ]]; then
+if [[ ! -d $checkout/.git ]]; then
   mkdir -p "$BUILD_DIR"
-  git init -q "$src"
-  git -C "$src" remote add origin https://github.com/NVIDIA/attestation-sdk.git
+  git init -q "$checkout"
+  git -C "$checkout" remote add origin https://github.com/NVIDIA/attestation-sdk.git
 fi
-git -C "$src" fetch -q --depth=1 origin "$NVATTEST_REVISION"
-git -C "$src" checkout -q --detach FETCH_HEAD
-git -C "$src" reset -q --hard "$NVATTEST_REVISION"
+git -C "$checkout" fetch -q --depth=1 origin "$NVATTEST_REVISION"
+git -C "$checkout" checkout -q --detach FETCH_HEAD
+git -C "$checkout" reset -q --hard "$NVATTEST_REVISION"
+git -C "$checkout" clean -qfdx
+rm -rf "$canonical"
+mkdir -p "$canonical"
+cp -a "$checkout" "$src"
 patch -d "$src" -p1 --fuzz=0 < \
   "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/0001-validate-ocsp-response-freshness.patch"
 patch -d "$src" -p1 --fuzz=0 < \
   "$SELF/patches/nvattest/0001-pin-fetchcontent-inputs.patch"
 
 rm -rf "$build"
+export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$canonical=/usr/src/nvattest -fmacro-prefix-map=$canonical=/usr/src/nvattest"
+export CXXFLAGS="${CXXFLAGS:-} -O2 -g0 -ffile-prefix-map=$canonical=/usr/src/nvattest -fmacro-prefix-map=$canonical=/usr/src/nvattest"
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="--remap-path-prefix=$canonical=/usr/src/nvattest -C strip=debuginfo"
 cmake -S "$src/nv-attestation-cli" -B "$build" -G Ninja \
   -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF \
   -DNVAT_BUILD_TESTS=OFF -DNVAT_BUILD_SAMPLES=OFF \
@@ -32,8 +50,6 @@ cmake -S "$src/nv-attestation-cli" -B "$build" -G Ninja \
 install -m0644 \
   "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/regorus-ffi-Cargo.lock" \
   "$build/_deps/regorus-src/bindings/ffi/Cargo.lock"
-export CARGO_INCREMENTAL=0
-export RUSTFLAGS="--remap-path-prefix=$BUILD_DIR=/usr/src/nvattest -C strip=debuginfo"
 cmake --build "$build" -j"${JOBS:-$(nproc)}"
 cmp "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/regorus-ffi-Cargo.lock" \
   "$build/_deps/regorus-src/bindings/ffi/Cargo.lock"

os/mkosi/scripts/build-nvidia.sh

@@ -32,14 +32,15 @@ rm -rf "$src"
 chmod +x "$run"
 "$run" -x --target "$src" >/dev/null
 patch -d "$src" -p1 --fuzz=0 < "$SELF/patches/nvidia/0001-linux-7.1-drop-legacy-of-gpio-api.patch"
+module_map="-fdebug-prefix-map=$BUILD_DIR=/usr/src/nvidia -fmacro-prefix-map=$BUILD_DIR=/usr/src/nvidia -fdebug-prefix-map=$KERNEL_SRC=/usr/src/linux -fmacro-prefix-map=$KERNEL_SRC=/usr/src/linux -fdebug-prefix-map=$KERNEL_BUILD=/usr/src/linux-build -fmacro-prefix-map=$KERNEL_BUILD=/usr/src/linux-build"
 # Linux 7.1's gen-btf.sh cannot execute its space-containing PAHOLE wrapper
 # for external modules (the awk program is passed with literal quotes). Keep
 # BTF enabled for the kernel/in-tree modules, but omit it for NVIDIA's modules.
 make -C "$src/kernel-open" -j"${JOBS:-$(nproc)}" \
   SYSSRC="$KERNEL_SRC" SYSOUT="$KERNEL_BUILD" \
-  CONFIG_DEBUG_INFO_BTF_MODULES= modules
+  KCFLAGS="$module_map" CONFIG_DEBUG_INFO_BTF_MODULES= modules
 make -C "$src/kernel-open" SYSSRC="$KERNEL_SRC" SYSOUT="$KERNEL_BUILD" \
-  CONFIG_DEBUG_INFO_BTF_MODULES= INSTALL_MOD_PATH="$KERNEL_STAGE" modules_install
+  KCFLAGS="$module_map" CONFIG_DEBUG_INFO_BTF_MODULES= INSTALL_MOD_PATH="$KERNEL_STAGE" modules_install
 if [[ -d $KERNEL_STAGE/lib/modules ]]; then
   mkdir -p "$KERNEL_STAGE/usr/lib/modules"
   cp -a "$KERNEL_STAGE/lib/modules/." "$KERNEL_STAGE/usr/lib/modules/"

os/mkosi/scripts/build-ovmf.sh

@@ -6,6 +6,8 @@ set -euo pipefail
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
 BUILD_DIR=${1:?build directory required}
 OUT=${2:?output file required}
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
+OUT=$(realpath -m "$OUT")
 REV=fbe0805b2091393406952e84724188f8c1941837
 src="$BUILD_DIR/edk2"
 if [[ ! -d $src/.git ]]; then

os/mkosi/scripts/build-sysbox.sh

@@ -7,6 +7,8 @@ SELF="$ROOT/os/mkosi"
 source "$SELF/versions.env"
 B=${1:?build directory required}
 STAGE=${2:?rootfs staging tree required}
+B=$(realpath -m "$B")
+STAGE=$(realpath -m "$STAGE")
 checkout() {
   local url=$1 rev=$2 dir=$3
   if [[ ! -d $dir/.git ]]; then git init -q "$dir"; git -C "$dir" remote add origin "$url"; fi

os/mkosi/scripts/build-zfs.sh

@@ -13,7 +13,9 @@ KERNEL_STAGE=$(realpath -m "${5:?kernel staging tree required}")
 src="$B/zfs"
 export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$B=/usr/src/zfs"
 export CXXFLAGS="${CXXFLAGS:-} -O2 -g0 -ffile-prefix-map=$B=/usr/src/zfs"
-export KCFLAGS="${KCFLAGS:-} -fmacro-prefix-map=$B=/usr/src/zfs"
+export KCFLAGS="${KCFLAGS:-} -fdebug-prefix-map=$B=/usr/src/zfs -fmacro-prefix-map=$B=/usr/src/zfs -fdebug-prefix-map=$KERNEL_SRC=/usr/src/linux -fmacro-prefix-map=$KERNEL_SRC=/usr/src/linux -fdebug-prefix-map=$KERNEL_BUILD=/usr/src/linux-build -fmacro-prefix-map=$KERNEL_BUILD=/usr/src/linux-build"
+export KAFLAGS="${KAFLAGS:-} $KCFLAGS"
+export KCPPFLAGS="${KCPPFLAGS:-} $KCFLAGS"
 if [[ ! -d $src/.git ]]; then git init -q "$src"; git -C "$src" remote add origin https://github.com/openzfs/zfs.git; fi
 git -C "$src" fetch -q --depth=1 origin "$ZFS_REVISION"
 git -C "$src" checkout -q --detach FETCH_HEAD
@@ -27,8 +29,11 @@ mkdir -p "$B/build"
   --with-linux-obj="$KERNEL_BUILD" --with-config=all --enable-systemd \
   --disable-pyzfs --without-dracutdir --with-udevdir=/usr/lib/udev \
   --with-systemdunitdir=/usr/lib/systemd/system)
-make -C "$B/build" -j"${JOBS:-$(nproc)}"
-make -C "$B/build" DESTDIR="$ROOT_STAGE" install
+# Linux 7.1 drives pahole 1.25's nondeterministic optimized encoder for
+# external modules. Kernel/in-tree BTF remains enabled; omit only ZFS module
+# BTF, as is also required for NVIDIA's external modules on this toolchain.
+make -C "$B/build" -j"${JOBS:-$(nproc)}" CONFIG_DEBUG_INFO_BTF_MODULES=
+make -C "$B/build" CONFIG_DEBUG_INFO_BTF_MODULES= DESTDIR="$ROOT_STAGE" install
 # ZFS installs kernel modules below DESTDIR/lib; merge them into usr-merge.
 if [[ -d $ROOT_STAGE/lib/modules ]]; then
   mkdir -p "$KERNEL_STAGE/usr/lib/modules"

os/mkosi/scripts/make-release-artifacts.sh

@@ -18,13 +18,17 @@ install -m0644 "$kernel" "$OUT/files/bzImage"
 
 # Combined squashfs + dm-verity tree, matching Yocto's separate-hash=0 output.
 rootfs="$OUT/files/rootfs.squashfs.verity"
+# mksquashfs -noappend overwrites its filesystem but does not reliably remove
+# a longer dm-verity tail left by a previous invocation.
+truncate -s 0 "$rootfs"
 env -u SOURCE_DATE_EPOCH mksquashfs "$TREE" "$rootfs" -noappend -all-root -no-progress \
   -comp zstd -mkfs-time "$SOURCE_DATE_EPOCH" -all-time "$SOURCE_DATE_EPOCH" >/dev/null
 data_size=$(stat -c %s "$rootfs")
 data_size=$(( (data_size + 4095) / 4096 * 4096 ))
 truncate -s "$data_size" "$rootfs"
 verity_output=$(veritysetup format "$rootfs" "$rootfs" \
   --hash-offset="$data_size" --data-block-size=4096 --hash-block-size=4096 \
+  --uuid=00000000-0000-0000-0000-000000000000 \
   --salt="$(printf '%064x' 0)")
 root_hash=$(awk '/Root hash:/ {print $3}' <<<"$verity_output")
 [[ $root_hash =~ ^[0-9a-f]{64}$ ]] || { echo "failed to obtain verity root hash" >&2; exit 1; }