os: add experimental reproducible Debian/mkosi backend

.github/workflows/mkosi-build.yml

@@ -0,0 +1,120 @@
+# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Verify mkosi reproducibility
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: mkosi-build-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # These values were produced by two cache-disabled local builds. The job
+  # checks out this exact source revision so the embedded Git revision and
+  # SOURCE_DATE_EPOCH are identical. Refresh all three values together.
+  BASELINE_REVISION: 98df4d5403db271b72cfaae3be742a577f1a2668
+  BARE_SHA256: d36fb8862ba8ebe5c99e0481e4e3e67ffc12240950ad28ba8c24dd5c2be5756e
+  UKI_SHA256: ad07b150fa7541134984ae3de8a320506bc2e5a07893441d3ccb95f3ab5db0f2
+
+jobs:
+  cross-host:
+    name: Compare with local mkosi baseline
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      contents: read
+
+    steps:
+      # GitHub-hosted runners have enough CPU and memory for this build, but
+      # their preinstalled SDKs consume most of the available disk. Remove only
+      # those unused SDKs before checkout; the OS build itself remains hermetic.
+      - name: Reclaim runner disk space
+        run: |
+          sudo rm -rf \
+            /opt/ghc \
+            /opt/hostedtoolcache/CodeQL \
+            /usr/local/.ghcup \
+            /usr/local/lib/android \
+            /usr/local/share/boost \
+            /usr/share/dotnet
+          df -h /
+
+      - name: Checkout locally verified revision
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ env.BASELINE_REVISION }}
+          fetch-depth: 0
+
+      - name: Set up Python for mkosi
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+
+      - name: Install pinned mkosi
+        run: |
+          python3 -m venv "$RUNNER_TEMP/mkosi-venv"
+          "$RUNNER_TEMP/mkosi-venv/bin/pip" install --disable-pip-version-check \
+            'mkosi @ git+https://github.com/systemd/mkosi.git@84af20892b61c8e177e391f997ded8b4cb5514f2'
+          echo "$RUNNER_TEMP/mkosi-venv/bin" >> "$GITHUB_PATH"
+
+      - name: Install mkosi host dependencies
+        run: |
+          mapfile -t dependencies < <(mkosi --directory os/mkosi dependencies)
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends "${dependencies[@]}"
+
+      - name: Build without component cache
+        env:
+          # ubuntu-latest currently provides four vCPUs and 16 GiB of RAM.
+          # Do not oversubscribe either the CPU or compiler memory.
+          JOBS: '4'
+        run: |
+          test -z "${DSTACK_DEV_CACHE_DIR:-}"
+          # GitHub-hosted Ubuntu runners restrict unprivileged user namespaces.
+          # Run mkosi as root instead of weakening the runner's AppArmor policy.
+          # Keep setup-python out of the build PATH because Ubuntu's lddtree
+          # expects the distro Python and its python3-pyelftools module.
+          ci_bin="$RUNNER_TEMP/mkosi-ci-bin"
+          mkdir -p "$ci_bin"
+          ln -s "$(command -v mkosi)" "$ci_bin/mkosi"
+          sudo --set-home env \
+            "PATH=$ci_bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+            "JOBS=$JOBS" \
+            ./os/mkosi/build.sh image "$RUNNER_TEMP/mkosi-cross-host"
+          sudo chown -R "$USER:$USER" "$RUNNER_TEMP/mkosi-cross-host/out"
+
+      - name: Compare with local byte-for-byte baseline
+        run: |
+          cd "$RUNNER_TEMP/mkosi-cross-host/out/prod"
+          tar -xOf dstack-0.6.0.tar.gz dstack-0.6.0/sha256sum.txt
+          tar -xOf dstack-0.6.0.tar.gz dstack-0.6.0/metadata.json
+          mkdir "$RUNNER_TEMP/rootfs-inspect"
+          sudo unsquashfs -d "$RUNNER_TEMP/rootfs-inspect" -o 1048576 \
+            dstack-0.6.0/rootfs.img.parted.verity >/dev/null
+          sudo chown -R "$USER:$USER" "$RUNNER_TEMP/rootfs-inspect"
+          (cd "$RUNNER_TEMP/rootfs-inspect" && \
+            find . -type f -print0 | sort -z | xargs -0 sha256sum) \
+            > rootfs-files.sha256
+          (cd "$RUNNER_TEMP/rootfs-inspect" && \
+            find . -printf '%y %m %u:%g %p -> %l\n' | sort) \
+            > rootfs-tree.txt
+          sha256sum dstack-0.6.0.tar.gz dstack-0.6.0-uki.tar.gz \
+            | tee cross-host.sha256
+          printf '%s  %s\n' "$BARE_SHA256" dstack-0.6.0.tar.gz \
+            | sha256sum --check --strict
+          printf '%s  %s\n' "$UKI_SHA256" dstack-0.6.0-uki.tar.gz \
+            | sha256sum --check --strict
+
+      - name: Upload verification record
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mkosi-cross-host-sha256
+          path: |
+            ${{ runner.temp }}/mkosi-cross-host/out/prod/cross-host.sha256
+            ${{ runner.temp }}/mkosi-cross-host/out/prod/rootfs-files.sha256
+            ${{ runner.temp }}/mkosi-cross-host/out/prod/rootfs-tree.txt
+          retention-days: 30

REUSE.toml

@@ -174,6 +174,14 @@ precedence = "override"
 
 # Generated files
 
+# Experimental Debian/mkosi guest OS backend. Some native configuration
+# formats (JSON, apt, kernel config and gitignore) cannot carry SPDX headers.
+[[annotations]]
+path = "os/mkosi/**"
+SPDX-FileCopyrightText = "© 2026 Phala Network <dstack@phala.network>"
+SPDX-License-Identifier = "Apache-2.0"
+precedence = "override"
+
 [[annotations]]
 path = "dstack/kms/auth-eth/typechain-types/**"
 SPDX-FileCopyrightText = "NONE"

dstack/crates/build-info/src/lib.rs

@@ -15,16 +15,19 @@ pub use git_version::git_version as __git_version;
 #[macro_export]
 macro_rules! git_revision {
     () => {
-        $crate::__git_version!(
-            args = [
-                "--abbrev=20",
-                "--always",
-                "--dirty=-modified",
-                "--exclude=*"
-            ],
-            prefix = "git:",
-            fallback = "unknown"
-        )
+        match option_env!("DSTACK_GIT_REVISION") {
+            Some(revision) => revision,
+            None => $crate::__git_version!(
+                args = [
+                    "--abbrev=20",
+                    "--always",
+                    "--dirty=-modified",
+                    "--exclude=*"
+                ],
+                prefix = "git:",
+                fallback = "unknown"
+            ),
+        }
     };
 }
 

os/README.md

@@ -9,15 +9,17 @@ os/
 ├── common/rootfs/            # backend-independent guest payload
 ├── image/                    # backend-independent release assembler
 ├── spec/                     # versioned backend artifact contract
-└── yocto/                    # the currently implemented backend
+├── mkosi/                    # experimental pinned Debian/mkosi backend
+└── yocto/                    # production backend
     ├── deps/                 # external Yocto layers (git submodules)
     ├── layers/               # dstack-owned Yocto layers
     └── tools/                # Yocto-specific host workarounds
 ```
 
-A future backend can be added at `os/<backend>/` (for example `os/mkosi/`)
-without moving shared payload or duplicating release packaging. No mkosi
-backend is implemented today.
+A backend can be added at `os/<backend>/` without moving shared payload or
+duplicating release packaging. The mkosi backend is experimental but implements
+the same artifact-manifest and common release-assembly contract as Yocto; see
+[`mkosi/README.md`](mkosi/README.md) for its scope and acceptance criteria.
 
 ## Build
 

os/build.sh

@@ -13,6 +13,7 @@ Build dstack guest OS release artifacts with a selected backend.
 
 Available backends:
   yocto    Reproducible Yocto build (default)
+  mkosi    Experimental pinned Debian/mkosi build
 
 A backend lives at os/<name>/build.sh and implements the "image" action.
 The common image contract is documented in os/README.md.

os/image/assemble.sh

@@ -306,11 +306,18 @@ build_uki_disk_image() {
         tmp_dir=$(mktemp -d)
         trap 'rm -rf "$tmp_dir"' EXIT
 
-        # Create EFI filesystem with UKI as bootloader
+        # Fix both the FAT volume metadata and directory entry timestamps.
+        # mmd stamps new directories with wall-clock time, so populate the
+        # filesystem recursively from a normalized host-side tree instead.
         local efi_img=${tmp_dir}/efi.img
-        mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null
-        mmd -i "$efi_img" ::EFI ::EFI/BOOT
-        mcopy -i "$efi_img" "$uki_file" ::EFI/BOOT/BOOTX64.EFI
+        local efi_tree=${tmp_dir}/tree
+        mkdir -p "$efi_tree/EFI/BOOT"
+        cp "$uki_file" "$efi_tree/EFI/BOOT/BOOTX64.EFI"
+        find "$efi_tree" -print0 | xargs -0r touch --no-dereference \
+          --date="@${SOURCE_DATE_EPOCH:-0}"
+        mkfs.vfat --invariant -F 32 -n DSTACKEFI -C "$efi_img" \
+          $((efi_size_aligned / 1024)) >/dev/null
+        mcopy -smp -i "$efi_img" "$efi_tree/EFI" ::
 
         dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none
         dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none

os/mkosi/.gitignore

@@ -0,0 +1,6 @@
+build*/
+repro*/
+*.raw
+*.raw.zst
+mkosi.tools/
+mkosi.tools.manifest

os/mkosi/README.md

@@ -0,0 +1,136 @@
+# Experimental mkosi backend
+
+This is an experimental Debian/mkosi implementation of the dstack guest OS.
+It is not yet a replacement for the release Yocto backend. Its acceptance
+target is functional parity with the Yocto image, not merely release archive
+compatibility. `parity.json` is the machine-checked inventory used by a build.
+
+The backend builds the same dstack services plus the pinned Yocto component
+set: Linux 7.1, NVIDIA 595.58.03 (open modules, userspace, firmware, Fabric
+Manager and NSCQ), nvattest 2026.06.09 with the OCSP-freshness patch, OpenZFS
+2.4.0 with upstream Linux 6.19--7.1 compatibility backports, Sysbox 0.6.7,
+NVIDIA Container Toolkit, nerdctl, CNI plugins and stargz-snapshotter 0.18.2.
+
+## Reproducibility model
+
+`versions.env` pins the stable 7.x kernel, Rust 1.92.0 and Go 1.22.2
+archives by SHA-256 and selects an immutable Debian snapshot. All components
+are compiled by `mkosi.build` inside mkosi's build-package overlay; host
+Rust/Go/GCC binaries are never used. C/C++ compilers, linkers, headers and
+build systems come from that snapshot, while the Rust and Go distributions are
+installed under `/opt/dstack-toolchains` after checksum verification. A mkosi
+tools tree built from the same snapshot supplies image-construction tools.
+`Cargo.lock`, `--locked`, a fixed `SOURCE_DATE_EPOCH`, normalized file mtimes,
+fixed kernel build identity and mkosi's deterministic image construction close
+the remaining inputs. `DSTACK_SKIP_RUST=1` exists only for rootfs/kernel
+development and does **not** produce a functional guest.
+
+The custom kernel starts from `x86_64_defconfig`, then applies the reviewed
+`kernel.config` fragment. This is the practical upstream equivalent of Yocto's
+`linux-yocto-tiny` plus explicit features: disabling arbitrary defconfig
+symbols without resolving Kconfig dependencies would be less auditable. Both
+the TDX DMA patch and the ACPI BadAML/SystemMemory sandbox patch are reused
+verbatim from `meta-dstack`; patch fuzz is forbidden. The final `.config` is
+checked before compilation.
+
+## Build and acceptance
+
+Host compilation and image-assembly toolchains are not required. mkosi 26
+creates the pinned Debian build overlay containing all component compilers and
+headers. Its minimized `misc` tools-tree profile plus explicit packages supplies
+`lddtree`, squashfs, dm-verity, disk, archive and UKI tools. The repository's
+`dstack-mr` and the pinned `nitro-tpm-pcr-compute` revision are compiled by
+`mkosi.build`, exported only for `mkosi.postoutput`, and removed from the guest.
+The host needs mkosi's own dependencies and root privileges (or a working user
+namespace), but no Rust, Go, C or C++ compiler.
+
+```sh
+./os/mkosi/build.sh lint
+./os/build.sh --backend mkosi --build-dir "$PWD/os/mkosi/build"
+./os/mkosi/build.sh repro-check "$PWD/os/mkosi/repro"
+# QEMU smoke-test the assembled UKI disk (host OVMF path is distro-specific)
+qemu-system-x86_64 -machine q35 -m 2G -nographic \
+  -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd \
+  -drive if=virtio,format=raw,file=os/mkosi/build/out/prod/dstack-0.6.0/disk.raw
+```
+
+For local iteration only, `dev-image` enables a component-output cache:
+
+```sh
+DSTACK_DEV_CACHE_DIR="$HOME/.cache/dstack/mkosi-dev" \
+  ./os/mkosi/build.sh dev-image "$PWD/os/mkosi/build-dev"
+```
+
+The cache covers dstack Rust, image tools, the container stack, Sysbox, nvattest, the
+kernel build tree, NVIDIA, ZFS and both OVMF variants. Its key conservatively
+includes the inputs, tools, packages and component dependencies declared by
+each descriptor in `components/<name>/<name>.sh`, plus architecture, flavor and
+`SOURCE_DATE_EPOCH`. For linked worktrees, `build.sh` records the Git-owned and
+untracked source path inventory on the host, while file contents are hashed in
+the mkosi build root; Git metadata is never mounted into the sandbox.
+`build-components.sh` is intentionally only the ordered component list.
+Component install trees are merged with strict non-directory conflict
+detection. `image` and `repro-check` never calculate, read or write component
+cache keys. Release artifacts, Debian rootfs, dm-verity data and measurements
+are never cached.
+
+mkosi's `Incremental=`, `CacheDirectory=` and `BuildDirectory=` cover
+whole-image/rootfs and persistent-work-directory reuse; they do not provide
+independently keyed output trees for components or reject file collisions when
+those trees are installed. The small component layer only supplies those two
+missing policies. Source download, build, cache-key inputs and output ownership
+remain together under `components/<name>/`; production builds bypass the layer's
+archive cache completely.
+
+On a 16-job development host, a clean production work directory takes about
+27 minutes with warm package downloads; allow 30--45 minutes with cold network
+caches. `repro-check` performs two such builds sequentially.
+
+Acceptance means: the static contract passes; a disk with systemd-boot/UKI
+boots on x86_64 QEMU; `/proc/config.gz` contains the checked TDX/SNP, TPM,
+ACPI, dm-verity/crypt, virtio, container and hardening options; dstack services
+are enabled; and two clean builds compare byte-for-byte. The backend exports
+artifact-manifest schema v1 and delegates final assembly to
+`os/image/assemble.sh`, exactly like Yocto. Its output contains the same
+`dstack-0.6.0/` directory, bare-metal and UKI tarballs, partitioned combined
+squashfs/dm-verity image, metadata, measurements, checksums, kernel, initramfs,
+OVMF and UKI. Debian supplies the base userspace while the parity checker
+requires the Yocto-visible binaries, services, configuration, kernel modules
+and production/development separation before assembly is allowed to proceed.
+
+The firmware is not Debian's generic OVMF: `components/ovmf/ovmf-build.sh` builds the same
+EDK2 stable-202502 revision and `pre202505` TDX measurement layout selected by
+the Yocto recipe. A generic OVMF cannot be substituted because `dstack-mr`
+would produce invalid or unparseable TDX measurement material.
+
+## Native mkosi boundary
+
+The distribution snapshot, build-only packages, guest packages, profiles,
+source mounts, source-date epoch, tools tree, package cleanup, file removal,
+systemd presets, tmpfiles, service masks and the build/postinstall/finalize/
+postoutput/clean lifecycle are mkosi-native. The native tar output is replaced
+in `mkosi.postoutput` by the identically named Yocto-compatible archive, so no
+unrelated mkosi rootfs artifact escapes the staging directory.
+
+Only four project-specific mechanisms remain:
+
+1. `make-release-artifacts.sh` and `os/image/assemble.sh` implement the existing
+   combined squashfs/dm-verity layout, initramfs command-line protocol,
+   measurements and archive member contract. mkosi's repart/UKI formats would
+   change that external interface.
+2. The development-only component cache provides independently keyed output
+   trees. mkosi's `Incremental=`, `BuildDirectory=` and
+   `BuildSourcesEphemeral=buildcache` cache a whole image or mutable build tree,
+   not isolated component install outputs.
+3. `merge-component-trees.py` rejects conflicting component-owned rootfs paths;
+   mkosi's normal tree overlays intentionally use last-writer-wins semantics.
+4. `normalize-skeleton-modes.sh` maps regular skeleton files to Git's two
+   portable mode classes (0644 or 0755). mkosi correctly preserves source
+   modes, but Git worktrees on shared hosts can add group-write bits that are
+   not represented in the Git index and would otherwise change the rootfs.
+
+The tiny postinstall hook is also retained because mkosi's native `MachineId=`
+supports a UUID, `random`, or `uninitialized`, while the Yocto contract requires
+an existing but empty `/etc/machine-id`. Rust and Go distributions are pinned by
+version and SHA-256 inside the mkosi build overlay because Debian trixie's Rust
+package is too old for this workspace; they never come from the host.