feat(os): align mkosi backend with Yocto guest stack

Author: kvinwang

os/mkosi/README.md

@@ -1,9 +1,15 @@
 # Experimental mkosi backend
 
-This is a deliberately small Debian/mkosi implementation of the dstack guest
-OS. It is an experiment, not yet a replacement for the release Yocto backend.
-It tracks the Yocto image's confidential-computing, container, storage,
-network, TPM, SSH, chrony and dstack guest-service capabilities.
+This is an experimental Debian/mkosi implementation of the dstack guest OS.
+It is not yet a replacement for the release Yocto backend. Its acceptance
+target is functional parity with the Yocto image, not merely release archive
+compatibility. `parity.json` is the machine-checked inventory used by a build.
+
+The backend builds the same dstack services plus the pinned Yocto component
+set: Linux 7.1, NVIDIA 595.58.03 (open modules, userspace, firmware, Fabric
+Manager and NSCQ), nvattest 2026.06.09 with the OCSP-freshness patch, OpenZFS
+2.4.0 with upstream Linux 6.19--7.1 compatibility backports, Sysbox 0.6.7,
+NVIDIA Container Toolkit, nerdctl, CNI plugins and stargz-snapshotter 0.18.2.
 
 ## Reproducibility model
 
@@ -25,12 +31,15 @@ checked before compilation.
 
 ## Build and acceptance
 
-Host requirements include mkosi >= 26, systemd tools, a C/Rust kernel and EDK2
-build toolchain, `bc`, `bison`, `flex`, `nasm`, `iasl`, OpenSSL/ELF/UUID
-development headers, `patch`, `pax-utils` (`lddtree`), `squashfs-tools`,
-`cryptsetup`, `gdisk`, `dosfstools`, `mtools`, `curl`, `xz`, QEMU/KVM and root
-privileges (or a working user namespace). Full UKI release assembly also needs
-the pinned `nitro-tpm-pcr-compute` described by `os/image/assemble.sh`.
+Host requirements include mkosi >= 26, systemd tools, C/C++/Go/Rust kernel and
+EDK2 build toolchains, `autoconf`, `automake`, `libtool`, `bc`, `bison`,
+`flex`, `nasm`, `iasl`, and development headers for OpenSSL, ELF, UUID, udev,
+aio, attr, blkid, curl, seccomp and tirpc. BTF generation requires `pahole`
+(the `dwarves` package). Runtime build tools include `patch`,
+`pax-utils` (`lddtree`), `squashfs-tools`, `cryptsetup`, `gdisk`,
+`dosfstools`, `mtools`, `curl`, `xz`, QEMU/KVM and root privileges (or a
+working user namespace). Full UKI release assembly also needs the pinned
+`nitro-tpm-pcr-compute` described by `os/image/assemble.sh`.
 
 ```sh
 ./os/mkosi/build.sh lint
@@ -42,6 +51,10 @@ qemu-system-x86_64 -machine q35 -m 2G -nographic \
   -drive if=virtio,format=raw,file=os/mkosi/build/out/prod/dstack-0.6.0/disk.raw
 ```
 
+On a 16-job development host, a clean production work directory takes about
+17 minutes (measured 16m45s); allow 20--30 minutes with cold compiler and
+network caches. `repro-check` performs two such builds sequentially.
+
 Acceptance means: the static contract passes; a disk with systemd-boot/UKI
 boots on x86_64 QEMU; `/proc/config.gz` contains the checked TDX/SNP, TPM,
 ACPI, dm-verity/crypt, virtio, container and hardening options; dstack services
@@ -50,8 +63,9 @@ artifact-manifest schema v1 and delegates final assembly to
 `os/image/assemble.sh`, exactly like Yocto. Its output contains the same
 `dstack-0.6.0/` directory, bare-metal and UKI tarballs, partitioned combined
 squashfs/dm-verity image, metadata, measurements, checksums, kernel, initramfs,
-OVMF and UKI. Package contents differ because this backend uses Debian, but
-consumers see the same release format and partition labels.
+OVMF and UKI. Debian supplies the base userspace while the parity checker
+requires the Yocto-visible binaries, services, configuration, kernel modules
+and production/development separation before assembly is allowed to proceed.
 
 The firmware is not Debian's generic OVMF: `build-ovmf.sh` builds the same
 EDK2 stable-202502 revision and `pre202505` TDX measurement layout selected by

os/mkosi/build.sh

@@ -22,9 +22,26 @@ build_one() {
   local out=$1 work=$2 flavor=$3
   local stage="$work/rootfs-stage" kstage="$work/kernel-stage" tree="$work/rootfs"
   rm -rf "$work" "$out"; mkdir -p "$stage" "$kstage" "$out"
-  "$SELF/scripts/stage-rootfs.sh" "$stage"
+  "$SELF/scripts/stage-rootfs.sh" "$stage" "$flavor"
+  "$SELF/scripts/build-container-stack.sh" "$work/container-stack-build" "$stage"
+  "$SELF/scripts/build-sysbox.sh" "$work/sysbox-build" "$stage"
+  "$SELF/scripts/build-nvattest.sh" "$work/nvattest-build" "$stage"
   "$SELF/scripts/build-kernel.sh" "$work" "$kstage"
+  "$SELF/scripts/build-nvidia.sh" "$work/nvidia-build" \
+    "$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$stage" "$kstage"
+  "$SELF/scripts/build-zfs.sh" "$work/zfs-build" \
+    "$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$stage" "$kstage"
   "$SELF/scripts/build-ovmf.sh" "$work/ovmf-build" "$kstage/ovmf.fd"
+  # ExtraTrees is copied over Debian's usr-merged root where /bin, /sbin and
+  # /lib are symlinks. Normalize build systems (notably OpenZFS) that install
+  # into the legacy physical directories before handing the tree to mkosi.
+  for legacy in bin sbin lib lib64; do
+    if [[ -d $stage/$legacy && ! -L $stage/$legacy ]]; then
+      mkdir -p "$stage/usr/$legacy"
+      cp -a "$stage/$legacy/." "$stage/usr/$legacy/"
+      rm -rf "${stage:?}/$legacy"
+    fi
+  done
   cat > "$work/mkosi.local.conf" <<EOF
 [Content]
 ExtraTrees=$stage
@@ -43,13 +60,15 @@ EOF
   fi
   devargs=()
   if [[ $flavor == dev ]]; then
-    devargs+=(--package=strace --package=tcpdump --package=gdb --package=vim)
+    devargs+=(--package=strace --package=tcpdump --package=gdb --package=vim \
+      --package=openssh-server)
   fi
   mkosi --directory "$SELF" --force --extra-tree="$stage" "${devargs[@]}" \
     --format=directory --output-directory="$work" --output=rootfs \
     --compress-output=no --bootable=no build
   mkdir -p "$tree/usr/lib/modules"
   cp -a "$kstage/usr/lib/modules/." "$tree/usr/lib/modules/"
+  "$SELF/tests/check-parity.py" "$SELF/parity.json" "$tree" "$kstage" "$flavor"
   artifact_dir="$work/artifacts/$flavor"
   "$SELF/scripts/make-release-artifacts.sh" "$tree" "$kstage" "$artifact_dir" "$flavor"
   DIST_DIR="$out" \

os/mkosi/kernel.config

@@ -56,7 +56,7 @@ CONFIG_XFS_FS=m
 CONFIG_BTRFS_FS=m
 CONFIG_OVERLAY_FS=y
 CONFIG_OVERLAY_FS_METACOPY=y
-CONFIG_FUSE_FS=m
+CONFIG_FUSE_FS=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_TMPFS_XATTR=y
 CONFIG_TMPFS_POSIX_ACL=y
@@ -79,10 +79,10 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_BPF_JIT=y
 CONFIG_BRIDGE=m
 CONFIG_BRIDGE_NETFILTER=m
-CONFIG_VETH=m
-CONFIG_VLAN_8021Q=m
+CONFIG_VETH=y
+CONFIG_VLAN_8021Q=y
 CONFIG_TUN=m
-CONFIG_WIREGUARD=m
+CONFIG_WIREGUARD=y
 CONFIG_NETFILTER=y
 CONFIG_NETFILTER_ADVANCED=y
 CONFIG_NF_TABLES=m
@@ -118,3 +118,68 @@ CONFIG_SUSPEND=n
 CONFIG_BT=n
 CONFIG_WLAN=n
 CONFIG_MMC=n
+
+# Yocto dstack fragments and minimal guest-only hardware policy.
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_SERIAL_8250_PCI=y
+CONFIG_NR_CPUS=512
+CONFIG_DEBUG_INFO_BTF=y
+CONFIG_DEBUG_INFO_BTF_MODULES=y
+CONFIG_KPROBES=y
+CONFIG_HOTPLUG_PCI=n
+CONFIG_INPUT=n
+CONFIG_SCSI=n
+CONFIG_CUSE=y
+CONFIG_TCG_VTPM_PROXY=y
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NFT_NAT=m
+CONFIG_NFT_MASQ=m
+CONFIG_NFT_REDIR=m
+CONFIG_NFT_CT=m
+CONFIG_NFT_LOG=m
+CONFIG_NFT_LIMIT=m
+CONFIG_NFT_REJECT=m
+CONFIG_NFT_REJECT_INET=m
+CONFIG_NFT_HASH=m
+CONFIG_SECURITY_SELINUX=y
+CONFIG_BRIDGE_VLAN_FILTERING=y
+CONFIG_XFRM=y
+CONFIG_XFRM_USER=y
+CONFIG_XFRM_ALGO=y
+CONFIG_NETFILTER_XT_MATCH_BPF=y
+CONFIG_CRYPTO_SEQIV=y
+CONFIG_BTRFS_FS_POSIX_ACL=y
+CONFIG_BRIDGE_NF_EBTABLES=m
+CONFIG_NETFILTER_FAMILY_BRIDGE=y
+CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
+CONFIG_IP_SET_HASH_IP=m
+CONFIG_IP_SET_HASH_NET=m
+CONFIG_IP_SET_HASH_IPPORT=m
+CONFIG_IP_SET_HASH_IPPORTIP=m
+CONFIG_IP_SET_HASH_IPPORTNET=m
+CONFIG_IP_SET_HASH_NETPORT=m
+CONFIG_IP_SET_HASH_NETIFACE=m
+CONFIG_IP_SET_BITMAP_IP=m
+CONFIG_IP_SET_BITMAP_PORT=m
+CONFIG_IP_SET_LIST_SET=m
+CONFIG_NETFILTER_XT_SET=m
+CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m
+CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_CRYPTO_ECC=y
+CONFIG_CRYPTO_ECDSA=y
+CONFIG_CRYPTO_ECDH=y
+CONFIG_DRM_I915=n
+CONFIG_DRM_AMDGPU=n
+CONFIG_DRM_NOUVEAU=n
+CONFIG_SOUND=n
+CONFIG_MEDIA_SUPPORT=n
+CONFIG_WIRELESS=n
+CONFIG_USB_SUPPORT=n
+CONFIG_DEBUG_INFO=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
+CONFIG_VT=n
+CONFIG_ATA=n
+CONFIG_DRM_FBDEV_EMULATION=n
+CONFIG_EXPERT=y

os/mkosi/mkosi.conf

@@ -32,10 +32,10 @@ Packages=
         iptables
         nftables
         docker.io
+        docker-cli
         docker-compose
         containerd
         chrony
-        openssh-server
         wireguard-tools
         cryptsetup
         dmsetup
@@ -48,11 +48,12 @@ Packages=
         e2fsprogs
         gdisk
         parted
-        zfsutils-linux
         busybox-static
         cryptsetup-bin
         ovmf
         systemd-boot-efi
+        acpid
+        numactl
 SkeletonTrees=mkosi.skeleton
 PostInstallationScripts=mkosi.postinst
 KernelCommandLine=console=ttyS0,115200n8 console=tty0 panic=30 coherent_pool=64M

os/mkosi/mkosi.postinst

@@ -5,7 +5,8 @@ set -euo pipefail
 # mkosi executes post-installation scripts outside the image with BUILDROOT.
 B=${BUILDROOT:?}
 mkdir -p "$B/dev" "$B/proc" "$B/sys" "$B/run" "$B/tmp" "$B/dstack" \
-  "$B/etc/wireguard" "$B/var/lib/docker" "$B/var/lib/containerd"
+  "$B/etc/wireguard" "$B/var/lib/docker" "$B/var/lib/containerd" \
+  "$B/var/volatile" "$B/dstack/.host-shared"
 rm -f "$B/etc/machine-id" "$B/var/lib/dbus/machine-id"
 : > "$B/etc/machine-id"
 rm -rf "$B/var/cache/apt/"* "$B/var/lib/apt/lists/"* "$B/var/log/"*
@@ -14,8 +15,14 @@ ln -sfn dstack "$B/tapp"
 ln -sfn ../lib/systemd/systemd "$B/usr/sbin/init"
 
 systemctl --root="$B" enable systemd-networkd systemd-resolved chrony docker \
+    var-volatile.mount dstack-volatile-binds.service \
     dstack-firstboot.service dstack-prepare.service \
     dstack-guest-agent.socket dstack-guest-agent.service \
-    app-compose.service wg-checker.service
+    app-compose.service wg-checker.service nvidia-persistenced.service \
+    nvidia-fabricmanager.service containerd-stargz-grpc.service \
+    sysbox.service sysbox-fs.service sysbox-mgr.service
 systemctl --root="$B" mask getty@.service serial-getty@.service debug-shell.service \
     sleep.target suspend.target hibernate.target hybrid-sleep.target
+if [[ -x $B/usr/bin/dstack-tee-simulator ]]; then
+  systemctl --root="$B" enable dstack-tee-simulator.service
+fi

os/mkosi/mkosi.skeleton/etc/docker/daemon.json

@@ -1,5 +1,8 @@
 {
-  "iptables": true,
-  "ip6tables": true,
-  "storage-driver": "overlay2"
+  "log-driver": "json-file",
+  "log-opts": {"max-size": "100m", "max-file": "10"},
+  "runtimes": {
+    "nvidia": {"args": [], "path": "nvidia-container-runtime"},
+    "sysbox-runc": {"path": "/usr/bin/sysbox-runc"}
+  }
 }

os/mkosi/mkosi.skeleton/usr/bin/dstack-volatile-binds.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+for name in cache lib log spool tmp; do
+    source=/var/volatile/$name
+    target=/var/$name
+    mkdir -p "$source" "$target"
+    mountpoint -q "$target" || mount --bind "$source" "$target"
+done

os/mkosi/mkosi.skeleton/usr/lib/systemd/system/dstack-volatile-binds.service

@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: Apache-2.0
+[Unit]
+Description=Bind volatile state over the immutable root
+DefaultDependencies=no
+Requires=var-volatile.mount
+After=var-volatile.mount
+Before=local-fs.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/dstack-volatile-binds.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=local-fs.target

os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd-persistent-storage.service.d/10-volatile-state.conf

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+[Unit]
+Requires=dstack-volatile-binds.service
+After=dstack-volatile-binds.service

os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-resolved.service.d/10-var-volatile.conf

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+[Unit]
+After=systemd-tmpfiles-setup.service var-volatile.mount
+Requires=var-volatile.mount