diff --git a/.github/workflows/mkosi-build.yml b/.github/workflows/mkosi-build.yml new file mode 100644 index 000000000..6a077f675 --- /dev/null +++ b/.github/workflows/mkosi-build.yml @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# +# SPDX-License-Identifier: Apache-2.0 + +name: Verify mkosi reproducibility + +on: + workflow_dispatch: + +concurrency: + group: mkosi-build-${{ github.ref }} + cancel-in-progress: true + +env: + # These values were produced by two cache-disabled local builds. The job + # checks out this exact source revision so the embedded Git revision and + # SOURCE_DATE_EPOCH are identical. Refresh all three values together. + BASELINE_REVISION: 2611402ad7cadded2a044777f9542b5ef6579393 + BARE_SHA256: 630921781a37210ddc21104390d1b9c9da5d725541fa48b5c0e9f1e1b1a10af1 + UKI_SHA256: 0b40d5095589235c465dea3dd6c294f00bd90f1c1ccb5cf9fd8c8ff475df85f0 + +jobs: + cross-host: + name: Compare with local mkosi baseline + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + + steps: + # GitHub-hosted runners have enough CPU and memory for this build, but + # their preinstalled SDKs consume most of the available disk. Remove only + # those unused SDKs before checkout; the OS build itself remains hermetic. + - name: Reclaim runner disk space + run: | + sudo rm -rf \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/boost \ + /usr/share/dotnet + df -h / + + - name: Checkout locally verified revision + uses: actions/checkout@v5 + with: + ref: ${{ env.BASELINE_REVISION }} + fetch-depth: 0 + + - name: Set up Python for mkosi + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install pinned mkosi + run: | + python3 -m venv "$RUNNER_TEMP/mkosi-venv" + "$RUNNER_TEMP/mkosi-venv/bin/pip" install --disable-pip-version-check \ + 'mkosi @ git+https://github.com/systemd/mkosi.git@84af20892b61c8e177e391f997ded8b4cb5514f2' + echo "$RUNNER_TEMP/mkosi-venv/bin" >> "$GITHUB_PATH" + + - name: Install mkosi host dependencies + run: | + mapfile -t dependencies < <(mkosi --directory os/mkosi dependencies) + sudo apt-get update + sudo apt-get install --yes --no-install-recommends "${dependencies[@]}" + + - name: Build without component cache + env: + # ubuntu-latest currently provides four vCPUs and 16 GiB of RAM. + # Do not oversubscribe either the CPU or compiler memory. + JOBS: '4' + run: | + test -z "${DSTACK_DEV_CACHE_DIR:-}" + # GitHub-hosted Ubuntu runners restrict unprivileged user namespaces. + # Run mkosi as root instead of weakening the runner's AppArmor policy. + # Keep setup-python out of the build PATH because Ubuntu's lddtree + # expects the distro Python and its python3-pyelftools module. + ci_bin="$RUNNER_TEMP/mkosi-ci-bin" + mkdir -p "$ci_bin" + ln -s "$(command -v mkosi)" "$ci_bin/mkosi" + sudo --set-home env \ + "PATH=$ci_bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + "JOBS=$JOBS" \ + ./os/mkosi/build.sh image "$RUNNER_TEMP/mkosi-cross-host" + sudo chown -R "$USER:$USER" "$RUNNER_TEMP/mkosi-cross-host/out" + + - name: Compare with local byte-for-byte baseline + run: | + cd "$RUNNER_TEMP/mkosi-cross-host/out/prod" + tar -xOf dstack-0.6.0.tar.gz dstack-0.6.0/sha256sum.txt + tar -xOf dstack-0.6.0.tar.gz dstack-0.6.0/metadata.json + sha256sum dstack-0.6.0.tar.gz dstack-0.6.0-uki.tar.gz \ + | tee cross-host.sha256 + printf '%s %s\n' "$BARE_SHA256" dstack-0.6.0.tar.gz \ + | sha256sum --check --strict + printf '%s %s\n' "$UKI_SHA256" dstack-0.6.0-uki.tar.gz \ + | sha256sum --check --strict + + - name: Upload verification record + if: always() + uses: actions/upload-artifact@v4 + with: + name: mkosi-cross-host-sha256 + path: ${{ runner.temp }}/mkosi-cross-host/out/prod/cross-host.sha256 + retention-days: 30 diff --git a/REUSE.toml b/REUSE.toml index 52dd38741..11309aea9 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -174,6 +174,14 @@ precedence = "override" # Generated files +# Experimental Debian/mkosi guest OS backend. Some native configuration +# formats (JSON, apt, kernel config and gitignore) cannot carry SPDX headers. +[[annotations]] +path = "os/mkosi/**" +SPDX-FileCopyrightText = "© 2026 Phala Network " +SPDX-License-Identifier = "Apache-2.0" +precedence = "override" + [[annotations]] path = "dstack/kms/auth-eth/typechain-types/**" SPDX-FileCopyrightText = "NONE" diff --git a/dstack/crates/build-info/src/lib.rs b/dstack/crates/build-info/src/lib.rs index 0828359eb..9cb6791ca 100644 --- a/dstack/crates/build-info/src/lib.rs +++ b/dstack/crates/build-info/src/lib.rs @@ -15,16 +15,19 @@ pub use git_version::git_version as __git_version; #[macro_export] macro_rules! git_revision { () => { - $crate::__git_version!( - args = [ - "--abbrev=20", - "--always", - "--dirty=-modified", - "--exclude=*" - ], - prefix = "git:", - fallback = "unknown" - ) + match option_env!("DSTACK_GIT_REVISION") { + Some(revision) => revision, + None => $crate::__git_version!( + args = [ + "--abbrev=20", + "--always", + "--dirty=-modified", + "--exclude=*" + ], + prefix = "git:", + fallback = "unknown" + ), + } }; } diff --git a/os/README.md b/os/README.md index a880d343f..c7059a92b 100644 --- a/os/README.md +++ b/os/README.md @@ -9,15 +9,17 @@ os/ ├── common/rootfs/ # backend-independent guest payload ├── image/ # backend-independent release assembler ├── spec/ # versioned backend artifact contract -└── yocto/ # the currently implemented backend +├── mkosi/ # experimental pinned Debian/mkosi backend +└── yocto/ # production backend ├── deps/ # external Yocto layers (git submodules) ├── layers/ # dstack-owned Yocto layers └── tools/ # Yocto-specific host workarounds ``` -A future backend can be added at `os//` (for example `os/mkosi/`) -without moving shared payload or duplicating release packaging. No mkosi -backend is implemented today. +A backend can be added at `os//` without moving shared payload or +duplicating release packaging. The mkosi backend is experimental but implements +the same artifact-manifest and common release-assembly contract as Yocto; see +[`mkosi/README.md`](mkosi/README.md) for its scope and acceptance criteria. ## Build diff --git a/os/build.sh b/os/build.sh index 63254982b..59d878208 100755 --- a/os/build.sh +++ b/os/build.sh @@ -13,6 +13,7 @@ Build dstack guest OS release artifacts with a selected backend. Available backends: yocto Reproducible Yocto build (default) + mkosi Experimental pinned Debian/mkosi build A backend lives at os//build.sh and implements the "image" action. The common image contract is documented in os/README.md. diff --git a/os/image/assemble.sh b/os/image/assemble.sh index f45b24669..fcf4a65d7 100755 --- a/os/image/assemble.sh +++ b/os/image/assemble.sh @@ -306,11 +306,18 @@ build_uki_disk_image() { tmp_dir=$(mktemp -d) trap 'rm -rf "$tmp_dir"' EXIT - # Create EFI filesystem with UKI as bootloader + # Fix both the FAT volume metadata and directory entry timestamps. + # mmd stamps new directories with wall-clock time, so populate the + # filesystem recursively from a normalized host-side tree instead. local efi_img=${tmp_dir}/efi.img - mkfs.vfat -F 32 -n DSTACKEFI -C "$efi_img" $((efi_size_aligned / 1024)) >/dev/null - mmd -i "$efi_img" ::EFI ::EFI/BOOT - mcopy -i "$efi_img" "$uki_file" ::EFI/BOOT/BOOTX64.EFI + local efi_tree=${tmp_dir}/tree + mkdir -p "$efi_tree/EFI/BOOT" + cp "$uki_file" "$efi_tree/EFI/BOOT/BOOTX64.EFI" + find "$efi_tree" -print0 | xargs -0r touch --no-dereference \ + --date="@${SOURCE_DATE_EPOCH:-0}" + mkfs.vfat --invariant -F 32 -n DSTACKEFI -C "$efi_img" \ + $((efi_size_aligned / 1024)) >/dev/null + mcopy -smp -i "$efi_img" "$efi_tree/EFI" :: dd if="$efi_img" of="$disk_img" bs=$align seek=$((efi_start / align)) conv=notrunc status=none dd if="$rootfs_img" of="$disk_img" bs=$align seek=$((rootfs_start / align)) conv=notrunc status=none diff --git a/os/mkosi/.gitignore b/os/mkosi/.gitignore new file mode 100644 index 000000000..09198f6df --- /dev/null +++ b/os/mkosi/.gitignore @@ -0,0 +1,6 @@ +build*/ +repro*/ +*.raw +*.raw.zst +mkosi.tools/ +mkosi.tools.manifest diff --git a/os/mkosi/README.md b/os/mkosi/README.md new file mode 100644 index 000000000..95b794284 --- /dev/null +++ b/os/mkosi/README.md @@ -0,0 +1,136 @@ +# Experimental mkosi backend + +This is an experimental Debian/mkosi implementation of the dstack guest OS. +It is not yet a replacement for the release Yocto backend. Its acceptance +target is functional parity with the Yocto image, not merely release archive +compatibility. `parity.json` is the machine-checked inventory used by a build. + +The backend builds the same dstack services plus the pinned Yocto component +set: Linux 7.1, NVIDIA 595.58.03 (open modules, userspace, firmware, Fabric +Manager and NSCQ), nvattest 2026.06.09 with the OCSP-freshness patch, OpenZFS +2.4.0 with upstream Linux 6.19--7.1 compatibility backports, Sysbox 0.6.7, +NVIDIA Container Toolkit, nerdctl, CNI plugins and stargz-snapshotter 0.18.2. + +## Reproducibility model + +`versions.env` pins the stable 7.x kernel, Rust 1.92.0 and Go 1.22.2 +archives by SHA-256 and selects an immutable Debian snapshot. All components +are compiled by `mkosi.build` inside mkosi's build-package overlay; host +Rust/Go/GCC binaries are never used. C/C++ compilers, linkers, headers and +build systems come from that snapshot, while the Rust and Go distributions are +installed under `/opt/dstack-toolchains` after checksum verification. A mkosi +tools tree built from the same snapshot supplies image-construction tools. +`Cargo.lock`, `--locked`, a fixed `SOURCE_DATE_EPOCH`, normalized file mtimes, +fixed kernel build identity and mkosi's deterministic image construction close +the remaining inputs. `DSTACK_SKIP_RUST=1` exists only for rootfs/kernel +development and does **not** produce a functional guest. + +The custom kernel starts from `x86_64_defconfig`, then applies the reviewed +`kernel.config` fragment. This is the practical upstream equivalent of Yocto's +`linux-yocto-tiny` plus explicit features: disabling arbitrary defconfig +symbols without resolving Kconfig dependencies would be less auditable. Both +the TDX DMA patch and the ACPI BadAML/SystemMemory sandbox patch are reused +verbatim from `meta-dstack`; patch fuzz is forbidden. The final `.config` is +checked before compilation. + +## Build and acceptance + +Host compilation and image-assembly toolchains are not required. mkosi 26 +creates the pinned Debian build overlay containing all component compilers and +headers. Its minimized `misc` tools-tree profile plus explicit packages supplies +`lddtree`, squashfs, dm-verity, disk, archive and UKI tools. The repository's +`dstack-mr` and the pinned `nitro-tpm-pcr-compute` revision are compiled by +`mkosi.build`, exported only for `mkosi.postoutput`, and removed from the guest. +The host needs mkosi's own dependencies and root privileges (or a working user +namespace), but no Rust, Go, C or C++ compiler. + +```sh +./os/mkosi/build.sh lint +./os/build.sh --backend mkosi --build-dir "$PWD/os/mkosi/build" +./os/mkosi/build.sh repro-check "$PWD/os/mkosi/repro" +# QEMU smoke-test the assembled UKI disk (host OVMF path is distro-specific) +qemu-system-x86_64 -machine q35 -m 2G -nographic \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd \ + -drive if=virtio,format=raw,file=os/mkosi/build/out/prod/dstack-0.6.0/disk.raw +``` + +For local iteration only, `dev-image` enables a component-output cache: + +```sh +DSTACK_DEV_CACHE_DIR="$HOME/.cache/dstack/mkosi-dev" \ + ./os/mkosi/build.sh dev-image "$PWD/os/mkosi/build-dev" +``` + +The cache covers dstack Rust, image tools, the container stack, Sysbox, nvattest, the +kernel build tree, NVIDIA, ZFS and both OVMF variants. Its key conservatively +includes the inputs, tools, packages and component dependencies declared by +each descriptor in `components//.sh`, plus architecture, flavor and +`SOURCE_DATE_EPOCH`. For linked worktrees, `build.sh` records the Git-owned and +untracked source path inventory on the host, while file contents are hashed in +the mkosi build root; Git metadata is never mounted into the sandbox. +`build-components.sh` is intentionally only the ordered component list. +Component install trees are merged with strict non-directory conflict +detection. `image` and `repro-check` never calculate, read or write component +cache keys. Release artifacts, Debian rootfs, dm-verity data and measurements +are never cached. + +mkosi's `Incremental=`, `CacheDirectory=` and `BuildDirectory=` cover +whole-image/rootfs and persistent-work-directory reuse; they do not provide +independently keyed output trees for components or reject file collisions when +those trees are installed. The small component layer only supplies those two +missing policies. Source download, build, cache-key inputs and output ownership +remain together under `components//`; production builds bypass the layer's +archive cache completely. + +On a 16-job development host, a clean production work directory takes about +27 minutes with warm package downloads; allow 30--45 minutes with cold network +caches. `repro-check` performs two such builds sequentially. + +Acceptance means: the static contract passes; a disk with systemd-boot/UKI +boots on x86_64 QEMU; `/proc/config.gz` contains the checked TDX/SNP, TPM, +ACPI, dm-verity/crypt, virtio, container and hardening options; dstack services +are enabled; and two clean builds compare byte-for-byte. The backend exports +artifact-manifest schema v1 and delegates final assembly to +`os/image/assemble.sh`, exactly like Yocto. Its output contains the same +`dstack-0.6.0/` directory, bare-metal and UKI tarballs, partitioned combined +squashfs/dm-verity image, metadata, measurements, checksums, kernel, initramfs, +OVMF and UKI. Debian supplies the base userspace while the parity checker +requires the Yocto-visible binaries, services, configuration, kernel modules +and production/development separation before assembly is allowed to proceed. + +The firmware is not Debian's generic OVMF: `components/ovmf/ovmf-build.sh` builds the same +EDK2 stable-202502 revision and `pre202505` TDX measurement layout selected by +the Yocto recipe. A generic OVMF cannot be substituted because `dstack-mr` +would produce invalid or unparseable TDX measurement material. + +## Native mkosi boundary + +The distribution snapshot, build-only packages, guest packages, profiles, +source mounts, source-date epoch, tools tree, package cleanup, file removal, +systemd presets, tmpfiles, service masks and the build/postinstall/finalize/ +postoutput/clean lifecycle are mkosi-native. The native tar output is replaced +in `mkosi.postoutput` by the identically named Yocto-compatible archive, so no +unrelated mkosi rootfs artifact escapes the staging directory. + +Only four project-specific mechanisms remain: + +1. `make-release-artifacts.sh` and `os/image/assemble.sh` implement the existing + combined squashfs/dm-verity layout, initramfs command-line protocol, + measurements and archive member contract. mkosi's repart/UKI formats would + change that external interface. +2. The development-only component cache provides independently keyed output + trees. mkosi's `Incremental=`, `BuildDirectory=` and + `BuildSourcesEphemeral=buildcache` cache a whole image or mutable build tree, + not isolated component install outputs. +3. `merge-component-trees.py` rejects conflicting component-owned rootfs paths; + mkosi's normal tree overlays intentionally use last-writer-wins semantics. +4. `normalize-skeleton-modes.sh` maps regular skeleton files to Git's two + portable mode classes (0644 or 0755). mkosi correctly preserves source + modes, but Git worktrees on shared hosts can add group-write bits that are + not represented in the Git index and would otherwise change the rootfs. + +The tiny postinstall hook is also retained because mkosi's native `MachineId=` +supports a UUID, `random`, or `uninitialized`, while the Yocto contract requires +an existing but empty `/etc/machine-id`. Rust and Go distributions are pinned by +version and SHA-256 inside the mkosi build overlay because Debian trixie's Rust +package is too old for this workspace; they never come from the host. diff --git a/os/mkosi/build.sh b/os/mkosi/build.sh new file mode 100755 index 000000000..820e0ff8a --- /dev/null +++ b/os/mkosi/build.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +umask 0022 +SELF=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ROOT=$(cd "$SELF/../.." && pwd) +# shellcheck source=/dev/null +source "$SELF/versions.env" +FLAVORS=${FLAVORS:-prod} +action=${1:-image} +BUILD_DIR=${2:-$SELF/build} +BUILD_DIR=$(realpath -m "$BUILD_DIR") +case "$action" in + image|dev-image|repro-check|lint) ;; + *) echo "Usage: $0 {image|dev-image|repro-check|lint} [build-dir]" >&2; exit 2 ;; +esac +if [[ $action == lint ]]; then exec "$SELF/tests/acceptance.sh"; fi +command -v mkosi >/dev/null || { echo "mkosi $MKOSI_VERSION is required" >&2; exit 1; } +actual=$(mkosi --version | awk '{print $2}' | cut -d. -f1) +[[ $actual == "$MKOSI_VERSION" ]] || { + echo "mkosi $MKOSI_VERSION required, found $actual" >&2; exit 1; +} +export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(git -C "$ROOT" log -1 --format=%ct)} +revision=$(git -C "$ROOT" rev-parse HEAD) +export DSTACK_GIT_REVISION=${DSTACK_GIT_REVISION:-git:${revision:0:20}} +export TZ=UTC LC_ALL=C +# A production invocation reconstructs the tools tree once from the immutable +# snapshot, then shares that read-only environment across all requested flavors +# or both legs of repro-check. +if [[ $action != dev-image ]]; then + mkosi --directory "$SELF" clean -f +fi + +build_one() { + local out=$1 flavor=$2 + mkdir -p "$out" + mkosi_args=( + --directory "$SELF" + --force + --output-directory="$out" + --profile="$flavor" + --source-date-epoch="$SOURCE_DATE_EPOCH" + --environment="DSTACK_COMPONENT_CACHE=$([[ $action == dev-image ]] && echo 1 || echo 0)" + --environment="DSTACK_GIT_REVISION=$DSTACK_GIT_REVISION" + --environment="DSTACK_SOURCE_REVISION=$revision" + --environment="JOBS=${JOBS:-$(nproc)}" + ) + if [[ $action == dev-image ]]; then + cache_root=${DSTACK_DEV_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/dstack/mkosi-dev} + mkdir -p "$cache_root" + "$SELF/scripts/write-source-manifest.py" "$ROOT" \ + "$cache_root/source-manifest" + mkosi_args+=(--build-sources="$cache_root:component-cache") + mkosi_args+=(--environment="DSTACK_SOURCE_MANIFEST=/work/src/component-cache/source-manifest") + fi + mkosi "${mkosi_args[@]}" build +} + +if [[ $action == image || $action == dev-image ]]; then + for flavor in $FLAVORS; do build_one "$BUILD_DIR/out/$flavor" "$flavor"; done + exit +fi +build_one "$BUILD_DIR/a" prod +build_one "$BUILD_DIR/b" prod +cmp "$BUILD_DIR/a/dstack-$DSTACK_VERSION.tar.gz" "$BUILD_DIR/b/dstack-$DSTACK_VERSION.tar.gz" +cmp "$BUILD_DIR/a/dstack-$DSTACK_VERSION-uki.tar.gz" \ + "$BUILD_DIR/b/dstack-$DSTACK_VERSION-uki.tar.gz" +echo 'reproducibility check passed' diff --git a/os/mkosi/components/container-stack/container-stack-build.sh b/os/mkosi/components/container-stack/container-stack-build.sh new file mode 100755 index 000000000..1532cb4a3 --- /dev/null +++ b/os/mkosi/components/container-stack/container-stack-build.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +BUILD_DIR=${1:?build directory required} +STAGE=${2:?rootfs staging tree required} +BUILD_DIR=$(realpath -m "$BUILD_DIR") +STAGE=$(realpath -m "$STAGE") +mkdir -p "$BUILD_DIR/downloads" "$STAGE/usr/bin" + +checkout() { + local url=$1 rev=$2 dir=$3 + if [[ ! -d $dir/.git ]]; then + git init -q "$dir"; git -C "$dir" remote add origin "$url" + fi + git -C "$dir" fetch -q --depth=1 origin "$rev" + git -C "$dir" checkout -q --detach FETCH_HEAD + git -C "$dir" reset -q --hard "$rev" + git -C "$dir" clean -qfdx +} +fetch_sha256() { + local url=$1 sha=$2 output=$3 + [[ -f $output ]] || { curl -fL --retry 3 -o "$output.tmp" "$url"; mv "$output.tmp" "$output"; } + echo "$sha $output" | sha256sum -c --status || { echo "checksum mismatch: $output" >&2; exit 1; } +} + +lnc="$BUILD_DIR/libnvidia-container" +checkout https://github.com/NVIDIA/libnvidia-container.git "$LIBNVIDIA_CONTAINER_REVISION" "$lnc" +patch -d "$lnc" -p1 --fuzz=0 < \ + "$SELF/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch" +export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$BUILD_DIR=/usr/src/container-stack -fmacro-prefix-map=$BUILD_DIR=/usr/src/container-stack" +export LDFLAGS="${LDFLAGS:-} -Wl,--build-id=none" +export CGO_CFLAGS="$CFLAGS" +export GOFLAGS="${GOFLAGS:-} -trimpath -buildvcs=false" +make -C "$lnc" -j"${JOBS:-$(nproc)}" LIB_VERSION=1.18.1 \ + WITH_NVCGO=yes WITH_LIBELF=yes WITH_TIRPC=yes all +# Upstream adds a CRC of its unshipped split-debug file. The debug file is not +# installed, and its CRC varies with the clean build directory. +objcopy --remove-section=.gnu_debuglink "$lnc/libnvidia-container.so.1.18.1" +install -m0755 "$lnc/nvidia-container-cli" "$STAGE/usr/bin/" +install -Dm0755 "$lnc/libnvidia-container.so.1.18.1" "$STAGE/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.18.1" +ln -sfn libnvidia-container.so.1.18.1 "$STAGE/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1" + +nct="$BUILD_DIR/nvidia-container-toolkit" +checkout https://github.com/NVIDIA/nvidia-container-toolkit.git "$NVIDIA_CONTAINER_TOOLKIT_REVISION" "$nct" +# The pinned revision already contains the Yocto patch semantically, split by +# platform; fail if the required Linux linker flags ever disappear. +grep -q '#cgo linux LDFLAGS: -Wl,--export-dynamic' "$nct/internal/cuda/cuda.go" +make -C "$nct" -j"${JOBS:-$(nproc)}" cmds +for bin in nvidia-container-runtime nvidia-container-runtime.cdi \ + nvidia-container-runtime-hook nvidia-container-runtime.legacy nvidia-ctk \ + nvidia-cdi-hook; do install -m0755 "$nct/$bin" "$STAGE/usr/bin/"; done +ln -sfn nvidia-container-runtime-hook "$STAGE/usr/bin/nvidia-container-toolkit" +install -Dm0644 "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit/files/config.toml" \ + "$STAGE/etc/nvidia-container-runtime/config.toml" + +nerd="$BUILD_DIR/downloads/nerdctl-$NERDCTL_VERSION.tar.gz" +fetch_sha256 "https://github.com/containerd/nerdctl/releases/download/v$NERDCTL_VERSION/nerdctl-$NERDCTL_VERSION-linux-amd64.tar.gz" "$NERDCTL_SHA256" "$nerd" +tar -xOf "$nerd" nerdctl > "$STAGE/usr/bin/nerdctl"; chmod 0755 "$STAGE/usr/bin/nerdctl" +cni="$BUILD_DIR/downloads/cni-$CNI_VERSION.tgz" +fetch_sha256 "https://github.com/containernetworking/plugins/releases/download/v$CNI_VERSION/cni-plugins-linux-amd64-v$CNI_VERSION.tgz" "$CNI_SHA256" "$cni" +mkdir -p "$STAGE/usr/lib/cni"; tar -C "$STAGE/usr/lib/cni" -xf "$cni" + +stargz="$BUILD_DIR/downloads/stargz-$STARGZ_VERSION.tar.gz" +fetch_sha256 "https://github.com/containerd/stargz-snapshotter/releases/download/v$STARGZ_VERSION/stargz-snapshotter-v$STARGZ_VERSION-linux-amd64.tar.gz" "$STARGZ_SHA256" "$stargz" +tar -xOf "$stargz" containerd-stargz-grpc > "$STAGE/usr/bin/containerd-stargz-grpc" +tar -xOf "$stargz" ctr-remote > "$STAGE/usr/bin/ctr-remote" +chmod 0755 "$STAGE/usr/bin/containerd-stargz-grpc" "$STAGE/usr/bin/ctr-remote" +install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-containers/stargz-snapshotter/files/containerd-stargz-grpc.service" \ + "$STAGE/usr/lib/systemd/system/containerd-stargz-grpc.service" +install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-containers/containerd-config/files/config.toml" \ + "$STAGE/etc/containerd/config.toml" +find "$STAGE" -print0 | xargs -0r touch -h -d "@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/container-stack/container-stack.sh b/os/mkosi/components/container-stack/container-stack.sh new file mode 100644 index 000000000..968752210 --- /dev/null +++ b/os/mkosi/components/container-stack/container-stack.sh @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=container-stack +COMPONENT_CACHE_PATHS=(component-stages/container-stack) +COMPONENT_ROOTFS_TREES=(component-stages/container-stack) +COMPONENT_KERNEL_TREES=() + +component_cache_key() { + key_value "$NVIDIA_CONTAINER_TOOLKIT_REVISION" "$LIBNVIDIA_CONTAINER_REVISION" \ + "$NERDCTL_VERSION" "$NERDCTL_SHA256" "$CNI_VERSION" "$CNI_SHA256" \ + "$STARGZ_VERSION" "$STARGZ_SHA256" + key_file "$COMPONENT_PATH/container-stack-build.sh" \ + "$COMPONENT_PATH/patches/0001-omit-prefix-map-from-build-flags.patch" + key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit \ + os/yocto/layers/meta-dstack/recipes-containers + key_tools gcc go + key_packages libelf-dev libtirpc-dev +} + +component_build() { + "$COMPONENT_PATH/container-stack-build.sh" "$WORK/container-stack-build" \ + "$WORK/component-stages/container-stack" +} diff --git a/os/mkosi/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch b/os/mkosi/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch new file mode 100644 index 000000000..ffc5f00b7 --- /dev/null +++ b/os/mkosi/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch @@ -0,0 +1,15 @@ +# Prefix-map arguments contain the intentionally different clean-build path. +# They must affect compilation without becoming runtime version metadata. +diff --git a/Makefile b/Makefile +index 1a5115c..87c501c 100644 +--- a/Makefile ++++ b/Makefile +@@ -203,7 +203,7 @@ DEPENDENCIES := $(BIN_OBJS:%.o=%.d) $(LIB_OBJS:%.lo=%.d) + $(BUILD_DEFS): + @printf '#define BUILD_DATE "%s"\n' '$(strip $(DATE))' >$(BUILD_DEFS) + @printf '#define BUILD_COMPILER "%s " __VERSION__\n' '$(notdir $(COMPILER))' >>$(BUILD_DEFS) +- @printf '#define BUILD_FLAGS "%s"\n' '$(strip $(CPPFLAGS) $(CFLAGS) $(LDFLAGS))' >>$(BUILD_DEFS) ++ @printf '#define BUILD_FLAGS "%s"\n' '$(strip $(filter-out -ffile-prefix-map=% -fmacro-prefix-map=%,$(CPPFLAGS) $(CFLAGS) $(LDFLAGS)))' >>$(BUILD_DEFS) + @printf '#define BUILD_REVISION "%s"\n' '$(strip $(REVISION))' >>$(BUILD_DEFS) + @printf '#define BUILD_PLATFORM "%s"\n' '$(strip $(PLATFORM))' >>$(BUILD_DEFS) + diff --git a/os/mkosi/components/dstack-rust/dstack-rust-build.sh b/os/mkosi/components/dstack-rust/dstack-rust-build.sh new file mode 100755 index 000000000..ad85be6b5 --- /dev/null +++ b/os/mkosi/components/dstack-rust/dstack-rust-build.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +DEST=${1:?staging tree required} +DEST=$(realpath -m "$DEST") +FLAVOR=${2:-prod} +install -d "$DEST/usr/bin" "$DEST/usr/lib/systemd/system" \ + "$DEST/etc/systemd/journald.conf.d" "$DEST/etc/systemd/resolved.conf.d" \ + "$DEST/etc/systemd/system/docker.service.d" \ + "$DEST/etc/systemd/system/containerd.service.d" "$DEST/etc/sysctl.d" +for s in dstack-prepare ephemeral-docker wg-checker app-compose; do + install -m0755 "$ROOT/os/common/rootfs/$s.sh" "$DEST/usr/bin/$s.sh" +done +install -m0644 "$ROOT/os/common/rootfs/"*.service \ + "$ROOT/os/common/rootfs/dstack-guest-agent.socket" "$DEST/usr/lib/systemd/system/" +install -m0644 "$ROOT/os/common/rootfs/journald.conf" "$DEST/etc/systemd/journald.conf.d/dstack.conf" +install -m0644 "$ROOT/os/common/rootfs/llmnr.conf" "$DEST/etc/systemd/resolved.conf.d/dstack.conf" +install -m0644 "$ROOT/os/common/rootfs/tdx-attest.conf" "$DEST/etc/" +install -m0644 "$ROOT/os/common/rootfs/sysctl.d/99-dstack.conf" "$DEST/etc/sysctl.d/" +install -m0644 "$ROOT/os/common/rootfs/docker.service.d/"* \ + "$DEST/etc/systemd/system/docker.service.d/" +install -m0644 "$ROOT/os/common/rootfs/containerd.service.d/"* \ + "$DEST/etc/systemd/system/containerd.service.d/" + +# Cargo.lock and --locked pin every registry/git dependency. The hermetic +# mkosi build root may fetch missing inputs but cannot update the lock file. +if [[ ${DSTACK_SKIP_RUST:-0} != 1 ]]; then + export CARGO_INCREMENTAL=0 CARGO_NET_OFFLINE=${CARGO_NET_OFFLINE:-false} + build_root=$(dirname "$DEST") + export CARGO_TARGET_DIR="$build_root/dstack-cargo-target" + # A single codegen unit avoids LLVM partition/scheduling differences across + # hosts with different CPU counts while retaining parallel crate builds. + export RUSTFLAGS="${RUSTFLAGS:-} --remap-path-prefix=$ROOT=/usr/src/dstack --remap-path-prefix=$build_root=/usr/src/dstack-build -C codegen-units=1 -C strip=debuginfo" + cargo build --locked --release --manifest-path "$ROOT/dstack/Cargo.toml" \ + -p dstack-guest-agent -p dstack-util + install -m0755 "$CARGO_TARGET_DIR/release/dstack-guest-agent" \ + "$CARGO_TARGET_DIR/release/dstack-util" "$DEST/usr/bin/" + if [[ $FLAVOR == dev ]]; then + cargo build --locked --release --manifest-path "$ROOT/dstack/Cargo.toml" \ + -p dstack-tee-simulator + install -m0755 "$CARGO_TARGET_DIR/release/dstack-tee-simulator" "$DEST/usr/bin/" + install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/dstack-tee-simulator.service" \ + "$DEST/usr/lib/systemd/system/dstack-tee-simulator.service" + install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/files/tee-simulator.conf" \ + "$DEST/etc/tee-simulator.conf" + fi +fi +find "$DEST" -print0 | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/dstack-rust/dstack-rust.sh b/os/mkosi/components/dstack-rust/dstack-rust.sh new file mode 100644 index 000000000..d153e5996 --- /dev/null +++ b/os/mkosi/components/dstack-rust/dstack-rust.sh @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=dstack-rust +COMPONENT_CACHE_PATHS=(component-stages/dstack-rust) +COMPONENT_ROOTFS_TREES=(component-stages/dstack-rust) +COMPONENT_KERNEL_TREES=() + +component_cache_key() { + key_file "$COMPONENT_PATH/dstack-rust-build.sh" + key_tree dstack os/common/rootfs + key_tools rustc cargo +} + +component_build() { + "$COMPONENT_PATH/dstack-rust-build.sh" "$WORK/component-stages/dstack-rust" "$FLAVOR" +} diff --git a/os/mkosi/components/image-tools/image-tools-build.sh b/os/mkosi/components/image-tools/image-tools-build.sh new file mode 100755 index 000000000..c8e214457 --- /dev/null +++ b/os/mkosi/components/image-tools/image-tools-build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +# shellcheck source=/dev/null +source "$ROOT/os/mkosi/versions.env" +TARGET=$(realpath -m "${1:?target directory required}") +DEST=$(realpath -m "${2:?staging tree required}") +BIN="$DEST/usr/lib/dstack/image-tools" + +mkdir -p "$TARGET" "$BIN" +export CARGO_TARGET_DIR="$TARGET/dstack" +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="${RUSTFLAGS:-} --remap-path-prefix=$ROOT=/usr/src/dstack --remap-path-prefix=$TARGET=/usr/src/image-tools -C codegen-units=1 -C strip=debuginfo" +cargo build --locked --release --manifest-path "$ROOT/dstack/Cargo.toml" \ + -p dstack-mr +install -m0755 "$CARGO_TARGET_DIR/release/dstack-mr" "$BIN/" + +export CARGO_TARGET_DIR="$TARGET/nitro-tpm" +cargo install --locked \ + --git https://github.com/aws/NitroTPM-Tools \ + --rev "$NITRO_TPM_TOOLS_REVISION" \ + --root "$TARGET/nitro-install" nitro-tpm-pcr-compute +install -m0755 "$TARGET/nitro-install/bin/nitro-tpm-pcr-compute" "$BIN/" + +find "$DEST" -print0 | xargs -0r touch --no-dereference \ + --date="@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/image-tools/image-tools.sh b/os/mkosi/components/image-tools/image-tools.sh new file mode 100644 index 000000000..bad7fd3a5 --- /dev/null +++ b/os/mkosi/components/image-tools/image-tools.sh @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=image-tools +COMPONENT_CACHE_PATHS=(component-stages/image-tools image-tools-target) +COMPONENT_ROOTFS_TREES=(component-stages/image-tools) +COMPONENT_KERNEL_TREES=() + +component_cache_key() { + key_value "$NITRO_TPM_TOOLS_REVISION" + key_file "$COMPONENT_PATH/image-tools-build.sh" + key_tree dstack/Cargo.toml dstack/Cargo.lock dstack/dstack-mr + key_tools rustc cargo + key_packages libssl-dev +} + +component_build() { + "$COMPONENT_PATH/image-tools-build.sh" \ + "$WORK/image-tools-target" "$WORK/component-stages/image-tools" +} diff --git a/os/mkosi/components/kernel/kernel-build.sh b/os/mkosi/components/kernel/kernel-build.sh new file mode 100755 index 000000000..64b049fc7 --- /dev/null +++ b/os/mkosi/components/kernel/kernel-build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +MKOSI_DIR="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$MKOSI_DIR/versions.env" +BUILD_DIR=${1:?build directory required} +STAGING=${2:?staging tree required} +BUILD_DIR=$(realpath -m "$BUILD_DIR") +STAGING=$(realpath -m "$STAGING") +JOBS=${JOBS:-$(nproc)} +export KBUILD_BUILD_TIMESTAMP="@${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH required}" +export KBUILD_BUILD_USER=dstack KBUILD_BUILD_HOST=reproducible +export KBUILD_BUILD_VERSION=1 +export KCONFIG_NOTIMESTAMP=1 +# Neither the source nor output directory may leak into DWARF/BTF or module +# metadata: repro-check deliberately builds in two differently named trees. +kernel_map="-fdebug-prefix-map=$BUILD_DIR=/usr/src/linux -fmacro-prefix-map=$BUILD_DIR=/usr/src/linux" +export KCFLAGS="${KCFLAGS:-} $kernel_map" +export KAFLAGS="${KAFLAGS:-} $kernel_map" +export KCPPFLAGS="${KCPPFLAGS:-} $kernel_map" + +mkdir -p "$BUILD_DIR/downloads" "$BUILD_DIR/kernel-build" "$STAGING" +# pahole 1.25 produces a different BTF type order when its encoder runs with +# Kbuild's parallel job count. Keep compilation parallel, but serialize BTF +# encoding so vmlinux and bzImage are byte-for-byte reproducible. +pahole_wrapper="$BUILD_DIR/pahole-reproducible" +cat > "$pahole_wrapper" <<'EOF' +#!/bin/bash +set -euo pipefail +args=() +for arg; do + # dwarves 1.25's parallel and optimized-function encoders both depend on + # unstable traversal order. Neither is needed for BTF correctness. + [[ $arg == -j* || $arg == --btf_gen_optimized ]] || args+=("$arg") +done +exec pahole -j1 "${args[@]}" +EOF +chmod 0755 "$pahole_wrapper" +tarball="$BUILD_DIR/downloads/linux-$KERNEL_VERSION.tar.xz" +if [[ ! -f $tarball ]]; then + curl --fail --location --retry 3 -o "$tarball.tmp" \ + "https://cdn.kernel.org/pub/linux/kernel/v7.x/linux-$KERNEL_VERSION.tar.xz" + mv "$tarball.tmp" "$tarball" +fi +echo "$KERNEL_SHA256 $tarball" | sha256sum --check --status || { + echo "kernel source checksum mismatch" >&2; exit 1; +} +src="$BUILD_DIR/linux-$KERNEL_VERSION" +rm -rf "$src" "$BUILD_DIR/kernel-build" "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack" +tar -C "$BUILD_DIR" --no-same-owner -xf "$tarball" +for patch in \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch"; do + patch -d "$src" -p1 --fuzz=0 < "$patch" +done + +make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" x86_64_defconfig +"$src/scripts/kconfig/merge_config.sh" -m -O "$BUILD_DIR/kernel-build" \ + "$BUILD_DIR/kernel-build/.config" "$MKOSI_DIR/components/kernel/kernel.config" +make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" olddefconfig +"$MKOSI_DIR/scripts/check-kernel-config.sh" "$BUILD_DIR/kernel-build/.config" +make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" -j"$JOBS" bzImage modules +make -C "$src" O="$BUILD_DIR/kernel-build" \ + PAHOLE="$pahole_wrapper" INSTALL_MOD_PATH="$STAGING" modules_install +# Debian is usr-merged: /lib is a symlink. Normalize modules into /usr/lib so +# ExtraTrees never tries to replace that symlink with a directory. +if [[ -d $STAGING/lib/modules ]]; then + mkdir -p "$STAGING/usr/lib/modules" + cp -a "$STAGING/lib/modules/." "$STAGING/usr/lib/modules/" + rm -rf "${STAGING:?}/lib" +fi +rm -f "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/build" \ + "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/source" +install -Dm0644 "$BUILD_DIR/kernel-build/arch/x86/boot/bzImage" \ + "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/vmlinuz" +install -Dm0644 "$BUILD_DIR/kernel-build/.config" \ + "$STAGING/usr/lib/modules/$KERNEL_VERSION-dstack/config" +find "$STAGING" -print0 | xargs -0r touch --no-dereference --date="@$SOURCE_DATE_EPOCH" diff --git a/os/mkosi/components/kernel/kernel.config b/os/mkosi/components/kernel/kernel.config new file mode 100644 index 000000000..8d687434b --- /dev/null +++ b/os/mkosi/components/kernel/kernel.config @@ -0,0 +1,185 @@ +# dstack 7.x kernel: x86_64 defconfig baseline plus explicitly audited guest features. +CONFIG_LOCALVERSION="-dstack" +CONFIG_LOCALVERSION_AUTO=n +CONFIG_MODULES=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_EFI=y +CONFIG_EFI_STUB=y +CONFIG_EFI_PARTITION=y +CONFIG_ACPI=y +CONFIG_ACPI_TABLE_UPGRADE=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT=y +CONFIG_KVM_GUEST=y +CONFIG_INTEL_TDX_GUEST=y +CONFIG_TDX_GUEST_DRIVER=y +CONFIG_TSM_REPORTS=y +CONFIG_AMD_MEM_ENCRYPT=y +CONFIG_SEV_GUEST=y +CONFIG_EFI_COCO_SECRET=y +CONFIG_EFI_SECRET=y +CONFIG_VMGENID=y +CONFIG_VIRT_DRIVERS=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS=y +CONFIG_TCG_CRB=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_NET=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_VIRTIO_VSOCKETS=y +CONFIG_VSOCKETS=y +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_9P_FS=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_CORE=y +CONFIG_NET_VENDOR_GOOGLE=y +CONFIG_GVE=y +CONFIG_NET_VENDOR_AMAZON=y +CONFIG_ENA_ETHERNET=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_MD=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_EXT4_FS=y +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_XFS_FS=m +CONFIG_BTRFS_FS=m +CONFIG_OVERLAY_FS=y +CONFIG_OVERLAY_FS_METACOPY=y +CONFIG_FUSE_FS=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_TMPFS_XATTR=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_QUOTA=y +CONFIG_QFMT_V2=y +CONFIG_CGROUPS=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_SCHED=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_BPF=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_BRIDGE=m +CONFIG_BRIDGE_NETFILTER=m +CONFIG_VETH=y +CONFIG_VLAN_8021Q=y +CONFIG_TUN=m +CONFIG_WIREGUARD=y +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_TABLES=m +CONFIG_NFT_COMPAT=m +CONFIG_NF_NAT=m +CONFIG_NETFILTER_XTABLES=m +CONFIG_IP_SET=m +CONFIG_IP_VS=m +CONFIG_IPVLAN=m +CONFIG_VXLAN=m +CONFIG_CONFIGFS_FS=y +CONFIG_SECURITY=y +CONFIG_SECURITY_DMESG_RESTRICT=y +CONFIG_HARDENED_USERCOPY=y +CONFIG_FORTIFY_SOURCE=y +CONFIG_INIT_ON_ALLOC_DEFAULT_ON=y +CONFIG_SLAB_FREELIST_RANDOM=y +CONFIG_SLAB_FREELIST_HARDENED=y +CONFIG_IO_STRICT_DEVMEM=y +CONFIG_STACKPROTECTOR_STRONG=y +CONFIG_STRICT_KERNEL_RWX=y +CONFIG_STRICT_MODULE_RWX=y +CONFIG_RANDOMIZE_BASE=y +CONFIG_CPU_MITIGATIONS=y +CONFIG_MITIGATION_PAGE_TABLE_ISOLATION=y +CONFIG_MITIGATION_RETPOLINE=y +CONFIG_LEGACY_TIOCSTI=n +CONFIG_MAGIC_SYSRQ=n +CONFIG_INET_ESP=n +CONFIG_INET6_ESP=n +CONFIG_AF_RXRPC=n +CONFIG_SUSPEND=n +CONFIG_BT=n +CONFIG_WLAN=n +CONFIG_MMC=n + +# Yocto dstack fragments and minimal guest-only hardware policy. +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_SERIAL_8250_PCI=y +CONFIG_NR_CPUS=512 +CONFIG_DEBUG_INFO_BTF=y +CONFIG_DEBUG_INFO_BTF_MODULES=y +CONFIG_KPROBES=y +CONFIG_HOTPLUG_PCI=n +CONFIG_INPUT=n +CONFIG_SCSI=n +CONFIG_CUSE=y +CONFIG_TCG_VTPM_PROXY=y +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NAT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_HASH=m +CONFIG_SECURITY_SELINUX=y +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_XFRM=y +CONFIG_XFRM_USER=y +CONFIG_XFRM_ALGO=y +CONFIG_NETFILTER_XT_MATCH_BPF=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_NETFILTER_FAMILY_BRIDGE=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_CRYPTO_ECC=y +CONFIG_CRYPTO_ECDSA=y +CONFIG_CRYPTO_ECDH=y +CONFIG_DRM_I915=n +CONFIG_DRM_AMDGPU=n +CONFIG_DRM_NOUVEAU=n +CONFIG_SOUND=n +CONFIG_MEDIA_SUPPORT=n +CONFIG_WIRELESS=n +CONFIG_USB_SUPPORT=n +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y +CONFIG_VT=n +CONFIG_ATA=n +CONFIG_DRM_FBDEV_EMULATION=n +CONFIG_EXPERT=y diff --git a/os/mkosi/components/kernel/kernel.sh b/os/mkosi/components/kernel/kernel.sh new file mode 100644 index 000000000..b98d23486 --- /dev/null +++ b/os/mkosi/components/kernel/kernel.sh @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=kernel +COMPONENT_CACHE_PATHS=("linux-$KERNEL_VERSION" kernel-build component-stages/kernel) +COMPONENT_ROOTFS_TREES=() +COMPONENT_KERNEL_TREES=(component-stages/kernel) + +component_cache_key() { + key_value "$KERNEL_VERSION" "$KERNEL_SHA256" + key_file "$COMPONENT_PATH/kernel-build.sh" "$COMPONENT_PATH/kernel.config" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch" + key_tools gcc ld make pahole + key_packages binutils dwarves bc bison flex libssl-dev libelf-dev +} + +component_build() { + "$COMPONENT_PATH/kernel-build.sh" "$WORK" "$WORK/component-stages/kernel" +} diff --git a/os/mkosi/components/nvattest/nvattest-build.sh b/os/mkosi/components/nvattest/nvattest-build.sh new file mode 100755 index 000000000..9612c392f --- /dev/null +++ b/os/mkosi/components/nvattest/nvattest-build.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +BUILD_DIR=${1:?build directory required} +STAGE=${2:?rootfs staging tree required} +BUILD_DIR=$(realpath -m "$BUILD_DIR") +STAGE=$(realpath -m "$STAGE") +checkout="$BUILD_DIR/attestation-sdk" +# Cargo includes the literal RUSTFLAGS in crate disambiguators. A prefix-map +# containing work-a/work-b would therefore change Rust symbol hashes even +# though the mapped paths are equal. Serialize this component in a stable, +# clean pathname and keep the per-build checkout as the verified input. +canonical=/var/tmp/dstack-mkosi-nvattest +exec 9>"$canonical.lock" +flock 9 +src="$canonical/attestation-sdk" +build="$canonical/build" + +if [[ ! -d $checkout/.git ]]; then + mkdir -p "$BUILD_DIR" + git init -q "$checkout" + git -C "$checkout" remote add origin https://github.com/NVIDIA/attestation-sdk.git +fi +git -C "$checkout" fetch -q --depth=1 origin "$NVATTEST_REVISION" +git -C "$checkout" checkout -q --detach FETCH_HEAD +git -C "$checkout" reset -q --hard "$NVATTEST_REVISION" +git -C "$checkout" clean -qfdx +rm -rf "$canonical" +mkdir -p "$canonical" +cp -a "$checkout" "$src" +patch -d "$src" -p1 --fuzz=0 < \ + "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/0001-validate-ocsp-response-freshness.patch" +patch -d "$src" -p1 --fuzz=0 < \ + "$SELF/components/nvattest/patches/0001-pin-fetchcontent-inputs.patch" + +rm -rf "$build" +export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$canonical=/usr/src/nvattest -fmacro-prefix-map=$canonical=/usr/src/nvattest" +export CXXFLAGS="${CXXFLAGS:-} -O2 -g0 -ffile-prefix-map=$canonical=/usr/src/nvattest -fmacro-prefix-map=$canonical=/usr/src/nvattest" +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="--remap-path-prefix=$canonical=/usr/src/nvattest -C strip=debuginfo" +cmake -S "$src/nv-attestation-cli" -B "$build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF \ + -DNVAT_BUILD_TESTS=OFF -DNVAT_BUILD_SAMPLES=OFF \ + -DCMAKE_SKIP_RPATH=ON -DFETCHCONTENT_FULLY_DISCONNECTED=OFF \ + -DUSE_SYSTEM_DEPS=ON +install -m0644 \ + "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/regorus-ffi-Cargo.lock" \ + "$build/_deps/regorus-src/bindings/ffi/Cargo.lock" +cmake --build "$build" -j"${JOBS:-$(nproc)}" +cmp "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/regorus-ffi-Cargo.lock" \ + "$build/_deps/regorus-src/bindings/ffi/Cargo.lock" +DESTDIR="$STAGE" cmake --install "$build" --prefix /usr + +install -Dm0644 \ + "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/10-nvidia-gpu-ordering.conf" \ + "$STAGE/usr/lib/systemd/system/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf" +install -Dm0644 \ + "$src/relying_party_policy_examples/allow_trust_outpost_ocsp.rego" \ + "$STAGE/usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego" +find "$STAGE/usr/lib" -maxdepth 1 -type f -name 'lib*.so*' ! -name 'libnvat.so*' -delete +find "$STAGE" -print0 | xargs -0r touch -h -d "@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/nvattest/nvattest.sh b/os/mkosi/components/nvattest/nvattest.sh new file mode 100644 index 000000000..8b5f1cb2b --- /dev/null +++ b/os/mkosi/components/nvattest/nvattest.sh @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=nvattest +COMPONENT_CACHE_PATHS=(component-stages/nvattest) +COMPONENT_ROOTFS_TREES=(component-stages/nvattest) +COMPONENT_KERNEL_TREES=() + +component_cache_key() { + key_value "$NVATTEST_VERSION" "$NVATTEST_REVISION" + key_file "$COMPONENT_PATH/nvattest-build.sh" \ + "$COMPONENT_PATH/patches/0001-pin-fetchcontent-inputs.patch" + key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvattest + key_tools gcc cmake ninja rustc cargo + key_packages libssl-dev libcurl4-openssl-dev +} + +component_build() { + "$COMPONENT_PATH/nvattest-build.sh" "$WORK/nvattest-build" \ + "$WORK/component-stages/nvattest" +} diff --git a/os/mkosi/components/nvattest/patches/0001-pin-fetchcontent-inputs.patch b/os/mkosi/components/nvattest/patches/0001-pin-fetchcontent-inputs.patch new file mode 100644 index 000000000..121ddd6d4 --- /dev/null +++ b/os/mkosi/components/nvattest/patches/0001-pin-fetchcontent-inputs.patch @@ -0,0 +1,82 @@ +# Pin every active FetchContent input used by the production build. +diff --git a/nv-attestation-cli/CMakeLists.txt b/nv-attestation-cli/CMakeLists.txt +index 5939e6b..f6c9060 100644 +--- a/nv-attestation-cli/CMakeLists.txt ++++ b/nv-attestation-cli/CMakeLists.txt +@@ -16,10 +16,10 @@ include(FetchContent) + FetchContent_Declare( + CLI11 + GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git +- GIT_TAG v2.6.1 ++ GIT_TAG bfffd37e1f804ca4fae1caae106935791696b6a9 + ) + +-FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz EXCLUDE_FROM_ALL) ++FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa EXCLUDE_FROM_ALL) + FetchContent_MakeAvailable(CLI11 json) + + add_executable(nvattest +@@ -55,12 +55,12 @@ if(USE_SYSTEM_NVAT) + FetchContent_Declare( + fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt.git +- GIT_TAG 10.2.1 ++ GIT_TAG e69e5f977d458f2650bb346dadf2ad30c5320281 + ) + FetchContent_Declare( + spdlog + GIT_REPOSITORY https://github.com/gabime/spdlog.git +- GIT_TAG v1.14.1 ++ GIT_TAG 27cb4c76708608465c413f6d0e6b8d99a4d84302 + ) + # Just populate to get source directories for headers (don't add to build) + FetchContent_GetProperties(fmt) +diff --git a/nv-attestation-sdk-cpp/CMakeLists.txt b/nv-attestation-sdk-cpp/CMakeLists.txt +index 583cff9..2cca673 100644 +--- a/nv-attestation-sdk-cpp/CMakeLists.txt ++++ b/nv-attestation-sdk-cpp/CMakeLists.txt +@@ -49,7 +49,7 @@ FetchContent_MakeAvailable(Corrosion) + FetchContent_Declare( + regorus + GIT_REPOSITORY https://github.com/microsoft/regorus.git +- GIT_TAG regorus-v0.4.0 ++ GIT_TAG c7bf460bc160c96e38048296e5708943d2e43909 + ) + FetchContent_MakeAvailable(regorus) + +@@ -71,7 +71,7 @@ set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "Disable building jwt-cpp examples") + FetchContent_Declare( + jwt-cpp + GIT_REPOSITORY https://github.com/Thalhammer/jwt-cpp.git +- GIT_TAG v0.7.1 ++ GIT_TAG e71e0c2d584baff06925bbb3aad683f677e4d498 + EXCLUDE_FROM_ALL + ) + +@@ -89,7 +89,7 @@ target_include_directories(own-jwt-cpp + # but it should work with c++14 also. so disable this warning. + target_compile_options(own-jwt-cpp INTERFACE -Wno-c++17-extensions) + +-FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz EXCLUDE_FROM_ALL) ++FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa EXCLUDE_FROM_ALL) + FetchContent_MakeAvailable(json) + + # Build fmt and spdlog as static libraries with PIC (needed for linking into shared libs) +@@ -106,7 +106,7 @@ set(FMT_INSTALL OFF CACHE BOOL "Don't install fmt headers" FORCE) + FetchContent_Declare( + fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt.git +- GIT_TAG 10.2.1 ++ GIT_TAG e69e5f977d458f2650bb346dadf2ad30c5320281 + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(fmt) +@@ -118,7 +118,7 @@ set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "Build spdlog as static library" FORCE) + FetchContent_Declare( + spdlog + GIT_REPOSITORY https://github.com/gabime/spdlog.git +- GIT_TAG v1.14.1 ++ GIT_TAG 27cb4c76708608465c413f6d0e6b8d99a4d84302 + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(spdlog) diff --git a/os/mkosi/components/nvidia/nvidia-build.sh b/os/mkosi/components/nvidia/nvidia-build.sh new file mode 100755 index 000000000..e39104c69 --- /dev/null +++ b/os/mkosi/components/nvidia/nvidia-build.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +BUILD_DIR=$(realpath -m "${1:?build directory required}") +KERNEL_SRC=$(realpath "${2:?kernel source required}") +KERNEL_BUILD=$(realpath "${3:?kernel build directory required}") +ROOT_STAGE=$(realpath -m "${4:?rootfs staging tree required}") +KERNEL_STAGE=$(realpath -m "${5:?kernel staging tree required}") +downloads="$BUILD_DIR/downloads" +src="$BUILD_DIR/NVIDIA-Linux-x86_64-$NVIDIA_VERSION" +mkdir -p "$downloads" "$ROOT_STAGE/usr/bin" "$ROOT_STAGE/usr/lib/x86_64-linux-gnu" + +fetch_sha256() { + local url=$1 sha=$2 output=$3 + if [[ ! -f $output ]]; then + curl -fL --retry 3 --retry-all-errors --connect-timeout 30 \ + --max-time 300 -o "$output.tmp" "$url" + mv "$output.tmp" "$output" + fi + echo "$sha $output" | sha256sum -c --status || { + echo "checksum mismatch: $output" >&2; exit 1; + } +} + +run="$downloads/NVIDIA-Linux-x86_64-$NVIDIA_VERSION.run" +fetch_sha256 "https://us.download.nvidia.com/tesla/$NVIDIA_VERSION/NVIDIA-Linux-x86_64-$NVIDIA_VERSION.run" \ + "$NVIDIA_RUN_SHA256" "$run" +rm -rf "$src" +chmod +x "$run" +"$run" -x --target "$src" >/dev/null +patch -d "$src" -p1 --fuzz=0 < "$SELF/components/nvidia/patches/0001-linux-7.1-drop-legacy-of-gpio-api.patch" +module_map="-fdebug-prefix-map=$BUILD_DIR=/usr/src/nvidia -fmacro-prefix-map=$BUILD_DIR=/usr/src/nvidia -fdebug-prefix-map=$KERNEL_SRC=/usr/src/linux -fmacro-prefix-map=$KERNEL_SRC=/usr/src/linux -fdebug-prefix-map=$KERNEL_BUILD=/usr/src/linux-build -fmacro-prefix-map=$KERNEL_BUILD=/usr/src/linux-build" +# Linux 7.1's gen-btf.sh cannot execute its space-containing PAHOLE wrapper +# for external modules (the awk program is passed with literal quotes). Keep +# BTF enabled for the kernel/in-tree modules, but omit it for NVIDIA's modules. +make -C "$src/kernel-open" -j"${JOBS:-$(nproc)}" \ + SYSSRC="$KERNEL_SRC" SYSOUT="$KERNEL_BUILD" \ + KCFLAGS="$module_map" CONFIG_DEBUG_INFO_BTF_MODULES= modules +make -C "$src/kernel-open" SYSSRC="$KERNEL_SRC" SYSOUT="$KERNEL_BUILD" \ + KCFLAGS="$module_map" CONFIG_DEBUG_INFO_BTF_MODULES= INSTALL_MOD_PATH="$KERNEL_STAGE" modules_install +if [[ -d $KERNEL_STAGE/lib/modules ]]; then + mkdir -p "$KERNEL_STAGE/usr/lib/modules" + cp -a "$KERNEL_STAGE/lib/modules/." "$KERNEL_STAGE/usr/lib/modules/" + rm -rf "${KERNEL_STAGE:?}/lib" +fi +ln -sfn usr/lib "$KERNEL_STAGE/lib" +depmod -b "$KERNEL_STAGE" "$KERNEL_VERSION-dstack" +rm -f "$KERNEL_STAGE/lib" + +libdir="$ROOT_STAGE/usr/lib/x86_64-linux-gnu" +for lib in libnvidia-ml libnvidia-allocator libnvidia-eglcore libnvidia-encode \ + libnvidia-glcore libnvidia-gpucomp libnvidia-ngx libnvidia-nvvm \ + libnvidia-opencl libnvidia-rtcore libnvidia-tls libnvidia-cfg \ + libnvidia-opticalflow libnvidia-glsi libnvidia-glvkspirv libcuda libnvcuvid \ + libnvidia-pkcs11-openssl3 libnvidia-pkcs11 libnvidia-ptxjitcompiler; do + file="$src/$lib.so.$NVIDIA_VERSION" + [[ -f $file ]] || { echo "missing NVIDIA library: $file" >&2; exit 1; } + install -m0755 "$file" "$libdir/" + ln -sfn "$lib.so.$NVIDIA_VERSION" "$libdir/$lib.so.1" + ln -sfn "$lib.so.1" "$libdir/$lib.so" +done +install -m0755 "$src/libnvidia-api.so.1" "$libdir/" +ln -sfn libnvidia-api.so.1 "$libdir/libnvidia-api.so" +ln -sfn "libnvidia-nvvm.so.$NVIDIA_VERSION" "$libdir/libnvidia-nvvm.so.4" +for bin in nvidia-smi nvidia-debugdump nvidia-persistenced nvidia-modprobe; do + install -m0755 "$src/$bin" "$ROOT_STAGE/usr/bin/$bin" +done +mkdir -p "$ROOT_STAGE/usr/lib/firmware/nvidia/$NVIDIA_VERSION" +cp -a "$src/firmware/." "$ROOT_STAGE/usr/lib/firmware/nvidia/$NVIDIA_VERSION/" + +fm="$downloads/fabricmanager-$NVIDIA_VERSION.tar.xz" +fetch_sha256 "https://developer.download.nvidia.com/compute/nvidia-driver/redist/fabricmanager/linux-x86_64/fabricmanager-linux-x86_64-$NVIDIA_VERSION-archive.tar.xz" \ + "$NVIDIA_FABRICMANAGER_SHA256" "$fm" +fm_dir="$BUILD_DIR/fabricmanager" +rm -rf "$fm_dir"; mkdir -p "$fm_dir"; tar -C "$fm_dir" --strip-components=1 -xf "$fm" +install -m0755 "$fm_dir/bin/"{nv-fabricmanager,nvidia-fabricmanager-start.sh,nvswitch-audit} "$ROOT_STAGE/usr/bin/" +install -m0755 "$fm_dir/lib/libnvfm.so.1" "$libdir/" +ln -sfn libnvfm.so.1 "$libdir/libnvfm.so" +install -Dm0644 "$fm_dir/systemd/nvidia-fabricmanager.service" "$ROOT_STAGE/usr/lib/systemd/system/nvidia-fabricmanager.service" +mkdir -p "$ROOT_STAGE/usr/share/nvidia/nvswitch" +cp -a "$fm_dir/share/nvidia/nvswitch/." "$ROOT_STAGE/usr/share/nvidia/nvswitch/" +install -m0644 "$fm_dir/etc/fabricmanager"*.cfg "$ROOT_STAGE/usr/share/nvidia/nvswitch/" + +nscq="$downloads/libnvidia-nscq-$NVIDIA_VERSION.tar.xz" +fetch_sha256 "https://developer.download.nvidia.com/compute/nvidia-driver/redist/libnvidia_nscq/linux-x86_64/libnvidia_nscq-linux-x86_64-$NVIDIA_VERSION-archive.tar.xz" \ + "$NVIDIA_NSCQ_SHA256" "$nscq" +nscq_dir="$BUILD_DIR/nscq" +rm -rf "$nscq_dir"; mkdir -p "$nscq_dir"; tar -C "$nscq_dir" --strip-components=1 -xf "$nscq" +install -m0755 "$nscq_dir/lib/libnvidia-nscq.so.$NVIDIA_VERSION" "$libdir/" +ln -sfn "libnvidia-nscq.so.$NVIDIA_VERSION" "$libdir/libnvidia-nscq.so.2.0" +ln -sfn libnvidia-nscq.so.2.0 "$libdir/libnvidia-nscq.so.2" +ln -sfn libnvidia-nscq.so.2 "$libdir/libnvidia-nscq.so" + +install -Dm0755 "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-gpu-detect" "$ROOT_STAGE/usr/bin/nvidia-gpu-detect" +install -Dm0644 "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-persistenced.service" "$ROOT_STAGE/usr/lib/systemd/system/nvidia-persistenced.service" +install -Dm0644 "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia.conf" "$ROOT_STAGE/etc/modules-load.d/nvidia.conf" +install -Dm0644 "$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvidia/files/nvidia-fabricmanager-nvswitch-condition.conf" "$ROOT_STAGE/etc/systemd/system/nvidia-fabricmanager.service.d/10-nvswitch-condition.conf" +find "$ROOT_STAGE" "$KERNEL_STAGE" -print0 | xargs -0r touch -h -d "@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/nvidia/nvidia.sh b/os/mkosi/components/nvidia/nvidia.sh new file mode 100644 index 000000000..f9f8f9093 --- /dev/null +++ b/os/mkosi/components/nvidia/nvidia.sh @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=nvidia +COMPONENT_CACHE_PATHS=(component-stages/nvidia-root component-stages/nvidia-kernel) +COMPONENT_ROOTFS_TREES=(component-stages/nvidia-root) +COMPONENT_KERNEL_TREES=(component-stages/nvidia-kernel) + +component_cache_key() { + key_dependency kernel + key_value "$NVIDIA_VERSION" "$NVIDIA_RUN_SHA256" \ + "$NVIDIA_FABRICMANAGER_SHA256" "$NVIDIA_NSCQ_SHA256" + key_file "$COMPONENT_PATH/nvidia-build.sh" \ + "$COMPONENT_PATH/patches/0001-linux-7.1-drop-legacy-of-gpio-api.patch" + key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvidia + key_tools gcc make +} + +component_build() { + "$COMPONENT_PATH/nvidia-build.sh" "$WORK/nvidia-build" \ + "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" \ + "$WORK/component-stages/nvidia-root" "$WORK/component-stages/nvidia-kernel" +} + +component_prepare_outputs() { + find "$WORK/component-stages/nvidia-kernel/usr/lib/modules" -maxdepth 2 \ + -type f -name 'modules.*' -delete +} diff --git a/os/mkosi/components/nvidia/patches/0001-linux-7.1-drop-legacy-of-gpio-api.patch b/os/mkosi/components/nvidia/patches/0001-linux-7.1-drop-legacy-of-gpio-api.patch new file mode 100644 index 000000000..779e10c66 --- /dev/null +++ b/os/mkosi/components/nvidia/patches/0001-linux-7.1-drop-legacy-of-gpio-api.patch @@ -0,0 +1,37 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: dstack contributors +Date: Wed, 22 Jul 2026 00:00:00 +0000 +Subject: [PATCH] kernel-open: support legacy OF GPIO removal in Linux 7.1 + +Linux 7.1 removed the integer-based OF GPIO consumer API and its +linux/of_gpio.h header. The x86_64 datacenter driver does not use the two +Tegra-only call sites, so provide the former failure result on 7.1+ while +retaining the upstream implementation for older kernels. + +Upstream-Status: Inappropriate [x86_64-only dstack guest] +--- + kernel-open/common/inc/nv-linux.h | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/kernel-open/common/inc/nv-linux.h b/kernel-open/common/inc/nv-linux.h +index 81d24d3..752dad0 100644 +--- a/kernel-open/common/inc/nv-linux.h ++++ b/kernel-open/common/inc/nv-linux.h +@@ -1726,7 +1726,15 @@ typedef enum + #include + #include + #include ++#if LINUX_VERSION_CODE >= KERNEL_VERSION(7, 1, 0) ++static inline int of_get_named_gpio(const struct device_node *np, ++ const char *propname, int index) ++{ ++ return -ENOENT; ++} ++#else + #include ++#endif + #include + #include + +-- +2.52.0 diff --git a/os/mkosi/components/ovmf/ovmf-build.sh b/os/mkosi/components/ovmf/ovmf-build.sh new file mode 100755 index 000000000..6a4a16b94 --- /dev/null +++ b/os/mkosi/components/ovmf/ovmf-build.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Build the same pinned Intel TDX OVMF generation consumed by dstack-mr. A +# stock distro OVMF is not compatible with dstack's TDX measurement format. +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +BUILD_DIR=${1:?build directory required} +OUT=${2:?output file required} +SEV_OUT=${3:?SEV output file required} +BUILD_DIR=$(realpath -m "$BUILD_DIR") +OUT=$(realpath -m "$OUT") +SEV_OUT=$(realpath -m "$SEV_OUT") +REV=fbe0805b2091393406952e84724188f8c1941837 +src="$BUILD_DIR/edk2" +if [[ ! -d $src/.git ]]; then + mkdir -p "$BUILD_DIR" + git init -q "$src" + git -C "$src" remote add origin https://github.com/tianocore/edk2.git +fi +git -C "$src" fetch -q --depth=1 origin "$REV" +git -C "$src" checkout -q --detach FETCH_HEAD +for attempt in 1 2 3; do + if git -C "$src" submodule update --init --recursive --depth=1; then + break + fi + (( attempt < 3 )) || { echo 'failed to fetch OVMF submodules' >&2; exit 1; } + git -C "$src" submodule sync --recursive + sleep $((attempt * 5)) +done +git -C "$src" reset -q --hard "$REV" +patch -d "$src" -p1 --forward --fuzz=0 < \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch" +patch -d "$src" -p1 --forward --fuzz=0 < \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch" + +export PYTHON_COMMAND=python3 +make -s -C "$src/BaseTools" -j"${JOBS:-$(nproc)}" +export WORKSPACE="$src" EDK_TOOLS_PATH="$src/BaseTools" +export PATH="$EDK_TOOLS_PATH/BinWrappers/PosixLike:$PATH" +# Yocto disables EDK2 LTO for deterministic link ordering. +sed -i -e 's/-flto/-fno-lto/g' -e 's/-DUSING_LTO//g' \ + "$src/BaseTools/Conf/tools_def.template" +sed -i 's/-Werror /-Werror -Wno-maybe-uninitialized -Wno-stringop-overflow /g' \ + "$src/BaseTools/Conf/tools_def.template" +rm -f "$src/Conf/tools_def.txt" +# shellcheck source=/dev/null +(cd "$src" && set +u && source edksetup.sh BaseTools >/dev/null && \ + build -a X64 -t GCC5 -b RELEASE -n "${JOBS:-$(nproc)}" \ + -p OvmfPkg/IntelTdx/IntelTdxX64.dsc) +install -Dm0644 "$src/Build/IntelTdx/RELEASE_GCC5/FV/OVMF.fd" "$OUT" +# shellcheck source=/dev/null +(cd "$src" && set +u && source edksetup.sh BaseTools >/dev/null && \ + build -a X64 -t GCC5 -b RELEASE -n "${JOBS:-$(nproc)}" \ + -p OvmfPkg/AmdSev/AmdSevX64.dsc) +install -Dm0644 "$src/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd" "$SEV_OUT" +touch -d "@${SOURCE_DATE_EPOCH:?}" "$OUT" "$SEV_OUT" diff --git a/os/mkosi/components/ovmf/ovmf.sh b/os/mkosi/components/ovmf/ovmf.sh new file mode 100644 index 000000000..c40e68f3e --- /dev/null +++ b/os/mkosi/components/ovmf/ovmf.sh @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=ovmf +COMPONENT_CACHE_PATHS=(component-stages/ovmf) +COMPONENT_ROOTFS_TREES=() +COMPONENT_KERNEL_TREES=(component-stages/ovmf) + +component_cache_key() { + key_file "$COMPONENT_PATH/ovmf-build.sh" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0006-OvmfPkg-AmdSev-drop-embedded-grub.patch" + key_tools gcc make python3 + key_packages nasm acpica-tools uuid-dev +} + +component_build() { + mkdir -p "$WORK/component-stages/ovmf/usr/lib/dstack/firmware" + "$COMPONENT_PATH/ovmf-build.sh" "$WORK/ovmf-build" \ + "$WORK/component-stages/ovmf/usr/lib/dstack/firmware/ovmf.fd" \ + "$WORK/component-stages/ovmf/usr/lib/dstack/firmware/ovmf-sev.fd" +} diff --git a/os/mkosi/components/sysbox/sysbox-build.sh b/os/mkosi/components/sysbox/sysbox-build.sh new file mode 100755 index 000000000..effc2e653 --- /dev/null +++ b/os/mkosi/components/sysbox/sysbox-build.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +B=${1:?build directory required} +STAGE=${2:?rootfs staging tree required} +B=$(realpath -m "$B") +STAGE=$(realpath -m "$STAGE") +checkout() { + local url=$1 rev=$2 dir=$3 + if [[ ! -d $dir/.git ]]; then git init -q "$dir"; git -C "$dir" remote add origin "$url"; fi + git -C "$dir" fetch -q --depth=1 origin "$rev" + git -C "$dir" checkout -q --detach FETCH_HEAD + git -C "$dir" reset -q --hard "$rev" +} +mkdir -p "$B" +checkout https://github.com/nestybox/sysbox.git "$SYSBOX_REVISION" "$B/sysbox" +checkout https://github.com/nestybox/sysbox-runc.git "$SYSBOX_RUNC_REVISION" "$B/sysbox-runc" +checkout https://github.com/nestybox/sysbox-fs.git "$SYSBOX_FS_REVISION" "$B/sysbox-fs" +checkout https://github.com/nestybox/sysbox-mgr.git "$SYSBOX_MGR_REVISION" "$B/sysbox-mgr" +checkout https://github.com/nestybox/sysbox-ipc.git "$SYSBOX_IPC_REVISION" "$B/sysbox-ipc" +checkout https://github.com/nestybox/sysbox-libs.git "$SYSBOX_LIBS_REVISION" "$B/sysbox-libs" +checkout https://github.com/Dstack-TEE/fuse.git "$SYSBOX_FUSE_REVISION" "$B/sysbox-fuse" +rm -rf "$B/sysbox-fs/bazil"; ln -s "$B/sysbox-fuse" "$B/sysbox-fs/bazil" +install -m0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxFsProtobuf.pb.go" \ + "$B/sysbox-ipc/sysboxFsGrpc/sysboxFsProtobuf/" +install -m0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/sysboxMgrProtobuf.pb.go" \ + "$B/sysbox-ipc/sysboxMgrGrpc/sysboxMgrProtobuf/" +for mod in sysbox-runc sysbox-fs sysbox-mgr; do (cd "$B/$mod" && go mod vendor); done +ldflags="-buildid= -s -w -X 'main.edition=Community Edition (CE)' -X main.version=$SYSBOX_VERSION -X main.commitId=$SYSBOX_REVISION -X 'main.builtAt=1970-01-01T00:00:00Z' -X main.builtBy=dstack" +(cd "$B/sysbox-runc" && go build -mod=vendor -buildvcs=false -trimpath -tags 'seccomp idmapped_mnt' -ldflags "$ldflags" -o "$B/sysbox-runc-bin" .) +(cd "$B/sysbox-fs" && go build -mod=vendor -buildvcs=false -trimpath -ldflags "$ldflags" -o "$B/sysbox-fs-bin" ./cmd/sysbox-fs) +(cd "$B/sysbox-mgr" && go build -mod=vendor -buildvcs=false -trimpath -tags idmapped_mnt -ldflags "$ldflags" -o "$B/sysbox-mgr-bin" .) +for bin in runc fs mgr; do install -Dm0755 "$B/sysbox-$bin-bin" "$STAGE/usr/bin/sysbox-$bin"; done +for f in sysbox.service sysbox-fs.service sysbox-mgr.service; do + install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/$f" "$STAGE/usr/lib/systemd/system/$f" +done +install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/99-sysbox-sysctl.conf" "$STAGE/etc/sysctl.d/99-sysbox-sysctl.conf" +install -Dm0644 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox/files/50-sysbox-mod.conf" "$STAGE/etc/modules-load.d/50-sysbox-mod.conf" +mkdir -p "$STAGE/var/lib/sysbox" +printf 'sysbox:100000:65536\n' >> "$STAGE/etc/subuid" +printf 'sysbox:100000:65536\n' >> "$STAGE/etc/subgid" +find "$STAGE" -print0 | xargs -0r touch -h -d "@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/sysbox/sysbox.sh b/os/mkosi/components/sysbox/sysbox.sh new file mode 100644 index 000000000..488c7bc4f --- /dev/null +++ b/os/mkosi/components/sysbox/sysbox.sh @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=sysbox +COMPONENT_CACHE_PATHS=(component-stages/sysbox) +COMPONENT_ROOTFS_TREES=(component-stages/sysbox) +COMPONENT_KERNEL_TREES=() + +component_cache_key() { + key_value "$SYSBOX_VERSION" "$SYSBOX_REVISION" "$SYSBOX_RUNC_REVISION" \ + "$SYSBOX_FS_REVISION" "$SYSBOX_MGR_REVISION" "$SYSBOX_IPC_REVISION" \ + "$SYSBOX_LIBS_REVISION" "$SYSBOX_FUSE_REVISION" + key_file "$COMPONENT_PATH/sysbox-build.sh" + key_tree os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox + key_tools go + key_packages libseccomp-dev +} + +component_build() { + "$COMPONENT_PATH/sysbox-build.sh" "$WORK/sysbox-build" \ + "$WORK/component-stages/sysbox" +} diff --git a/os/mkosi/components/zfs/patches/0001-linux-6.19-7.1-compat.patch b/os/mkosi/components/zfs/patches/0001-linux-6.19-7.1-compat.patch new file mode 100644 index 000000000..8e2cd7f07 --- /dev/null +++ b/os/mkosi/components/zfs/patches/0001-linux-6.19-7.1-compat.patch @@ -0,0 +1,937 @@ +# Backport of upstream OpenZFS Linux compatibility commits onto tag 2.4.0: +# 3c8665cb, 168023b, 204de94, d34fd6c, 0f608aa, d8c08a1, 8518e3e, +# 4155d15, d88d9c9, c18e8ba and a35e8d8. +diff --git a/META b/META +index 86d5f7f..b28eb2d 100644 +--- a/META ++++ b/META +@@ -6,5 +6,5 @@ Release: 1 + Release-Tags: relext + License: CDDL + Author: OpenZFS +-Linux-Maximum: 6.18 ++Linux-Maximum: 7.1 + Linux-Minimum: 4.18 +diff --git a/config/kernel-acl.m4 b/config/kernel-acl.m4 +index 3dbd979..4393e3a 100644 +--- a/config/kernel-acl.m4 ++++ b/config/kernel-acl.m4 +@@ -21,6 +21,35 @@ AC_DEFUN([ZFS_AC_KERNEL_POSIX_ACL_EQUIV_MODE_WANTS_UMODE_T], [ + ]) + ]) + ++dnl # ++dnl # 7.0 API change ++dnl # posix_acl_to_xattr() now allocates and returns the value. ++dnl # ++AC_DEFUN([ZFS_AC_KERNEL_SRC_POSIX_ACL_TO_XATTR_ALLOC], [ ++ ZFS_LINUX_TEST_SRC([posix_acl_to_xattr_alloc], [ ++ #include ++ #include ++ ], [ ++ struct user_namespace *ns = NULL; ++ struct posix_acl *acl = NULL; ++ size_t size = 0; ++ gfp_t gfp = 0; ++ void *xattr = NULL; ++ xattr = posix_acl_to_xattr(ns, acl, &size, gfp); ++ ]) ++]) ++ ++AC_DEFUN([ZFS_AC_KERNEL_POSIX_ACL_TO_XATTR_ALLOC], [ ++ AC_MSG_CHECKING([whether posix_acl_to_xattr() allocates its result]); ++ ZFS_LINUX_TEST_RESULT([posix_acl_to_xattr_alloc], [ ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_POSIX_ACL_TO_XATTR_ALLOC, 1, ++ [posix_acl_to_xattr() allocates its result]) ++ ], [ ++ AC_MSG_RESULT(no) ++ ]) ++]) ++ + dnl # + dnl # 3.1 API change, + dnl # Check if inode_operations contains the function get_acl +@@ -173,12 +202,14 @@ AC_DEFUN([ZFS_AC_KERNEL_INODE_OPERATIONS_SET_ACL], [ + + AC_DEFUN([ZFS_AC_KERNEL_SRC_ACL], [ + ZFS_AC_KERNEL_SRC_POSIX_ACL_EQUIV_MODE_WANTS_UMODE_T ++ ZFS_AC_KERNEL_SRC_POSIX_ACL_TO_XATTR_ALLOC + ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_GET_ACL + ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL + ]) + + AC_DEFUN([ZFS_AC_KERNEL_ACL], [ + ZFS_AC_KERNEL_POSIX_ACL_EQUIV_MODE_WANTS_UMODE_T ++ ZFS_AC_KERNEL_POSIX_ACL_TO_XATTR_ALLOC + ZFS_AC_KERNEL_INODE_OPERATIONS_GET_ACL + ZFS_AC_KERNEL_INODE_OPERATIONS_SET_ACL + ]) +diff --git a/config/kernel-blk-queue.m4 b/config/kernel-blk-queue.m4 +index cd2b143..c6479b5 100644 +--- a/config/kernel-blk-queue.m4 ++++ b/config/kernel-blk-queue.m4 +@@ -225,6 +225,30 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS], [ + ]) + ]) + ++dnl # ++dnl # 7.0 API change ++dnl # blk_queue_rot() replaces blk_queue_nonrot() (inverted meaning) ++dnl # ++AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE_ROT], [ ++ ZFS_LINUX_TEST_SRC([blk_queue_rot], [ ++ #include ++ ], [ ++ struct request_queue *q __attribute__ ((unused)) = NULL; ++ (void) blk_queue_rot(q); ++ ], []) ++]) ++ ++AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_ROT], [ ++ AC_MSG_CHECKING([whether blk_queue_rot() is available]) ++ ZFS_LINUX_TEST_RESULT([blk_queue_rot], [ ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_BLK_QUEUE_ROT, 1, ++ [blk_queue_rot() is available]) ++ ],[ ++ AC_MSG_RESULT(no) ++ ]) ++]) ++ + dnl # + dnl # 2.6.34 API change + dnl # blk_queue_max_segments() consolidates blk_queue_max_hw_segments() +@@ -278,6 +302,7 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_BLK_QUEUE], [ + ZFS_AC_KERNEL_SRC_BLK_QUEUE_SECURE_ERASE + ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_HW_SECTORS + ZFS_AC_KERNEL_SRC_BLK_QUEUE_MAX_SEGMENTS ++ ZFS_AC_KERNEL_SRC_BLK_QUEUE_ROT + ZFS_AC_KERNEL_SRC_BLK_MQ_RQ_HCTX + ]) + +@@ -290,5 +315,6 @@ AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE], [ + ZFS_AC_KERNEL_BLK_QUEUE_SECURE_ERASE + ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS + ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS ++ ZFS_AC_KERNEL_BLK_QUEUE_ROT + ZFS_AC_KERNEL_BLK_MQ_RQ_HCTX + ]) +diff --git a/config/kernel-copy-from-user-inatomic.m4 b/config/kernel-copy-from-user-inatomic.m4 +deleted file mode 100644 +index fec354b..0000000 +--- a/config/kernel-copy-from-user-inatomic.m4 ++++ /dev/null +@@ -1,29 +0,0 @@ +-dnl # +-dnl # On certain architectures `__copy_from_user_inatomic` +-dnl # is a GPL exported variable and cannot be used by OpenZFS. +-dnl # +- +-dnl # +-dnl # Checking if `__copy_from_user_inatomic` is available. +-dnl # +-AC_DEFUN([ZFS_AC_KERNEL_SRC___COPY_FROM_USER_INATOMIC], [ +- ZFS_LINUX_TEST_SRC([__copy_from_user_inatomic], [ +- #include +- ], [ +- int result __attribute__ ((unused)) = __copy_from_user_inatomic(NULL, NULL, 0); +- ], [], [ZFS_META_LICENSE]) +-]) +- +-AC_DEFUN([ZFS_AC_KERNEL___COPY_FROM_USER_INATOMIC], [ +- AC_MSG_CHECKING([whether __copy_from_user_inatomic is available]) +- ZFS_LINUX_TEST_RESULT([__copy_from_user_inatomic_license], [ +- AC_MSG_RESULT(yes) +- ], [ +- AC_MSG_RESULT(no) +- AC_MSG_ERROR([ +- *** The `__copy_from_user_inatomic()` Linux kernel function is +- *** incompatible with the CDDL license and will prevent the module +- *** linking stage from succeeding. OpenZFS cannot be compiled. +- ]) +- ]) +-]) +diff --git a/config/kernel-dentry-alias.m4 b/config/kernel-dentry-alias.m4 +new file mode 100644 +index 0000000..0baf1a0 +--- /dev/null ++++ b/config/kernel-dentry-alias.m4 +@@ -0,0 +1,32 @@ ++dnl # SPDX-License-Identifier: CDDL-1.0 ++dnl # ++dnl # 7.1 API change ++dnl # d_u union in struct dentry is now anonmymous, so d_alias must be ++dnl # named directly ++dnl # ++AC_DEFUN([ZFS_AC_KERNEL_SRC_DENTRY_ALIAS_D_U], [ ++ ZFS_LINUX_TEST_SRC([dentry_alias_d_u], [ ++ #include ++ #include ++ #include ++ ], [ ++ struct inode *inode __attribute__ ((unused)) = NULL; ++ struct dentry *dentry __attribute__ ((unused)) = NULL; ++ hlist_for_each_entry(dentry, &inode->i_dentry, ++ d_u.d_alias) { ++ d_drop(dentry); ++ } ++ ]) ++]) ++ ++AC_DEFUN([ZFS_AC_KERNEL_DENTRY_ALIAS_D_U], [ ++ AC_MSG_CHECKING([whether dentry aliases are in d_u member]) ++ ZFS_LINUX_TEST_RESULT([dentry_alias_d_u], [ ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_DENTRY_D_U_ALIASES, 1, ++ [dentry aliases are in d_u member]) ++ ],[ ++ AC_MSG_RESULT(no) ++ ]) ++]) ++ +diff --git a/config/kernel-filelock.m4 b/config/kernel-filelock.m4 +new file mode 100644 +index 0000000..5e8d7c7 +--- /dev/null ++++ b/config/kernel-filelock.m4 +@@ -0,0 +1,23 @@ ++dnl # SPDX-License-Identifier: CDDL-1.0 ++dnl # ++dnl # 6.3 API change ++dnl # locking support functions (eg generic_setlease) were moved out of ++dnl # linux/fs.h to linux/filelock.h ++dnl # ++AC_DEFUN([ZFS_AC_KERNEL_SRC_FILELOCK_HEADER], [ ++ ZFS_LINUX_TEST_SRC([filelock_header], [ ++ #include ++ #include ++ ], []) ++]) ++ ++AC_DEFUN([ZFS_AC_KERNEL_FILELOCK_HEADER], [ ++ AC_MSG_CHECKING([for standalone filelock header]) ++ ZFS_LINUX_TEST_RESULT([filelock_header], [ ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_FILELOCK_HEADER, 1, [linux/filelock.h exists]) ++ ], [ ++ AC_MSG_RESULT(no) ++ ]) ++]) ++ +diff --git a/config/kernel-fst-mount.m4 b/config/kernel-fst-mount.m4 +index 576f5f0..798acf1 100644 +--- a/config/kernel-fst-mount.m4 ++++ b/config/kernel-fst-mount.m4 +@@ -3,6 +3,10 @@ dnl # 2.6.38 API change + dnl # The .get_sb callback has been replaced by a .mount callback + dnl # in the file_system_type structure. + dnl # ++dnl # 7.0 API change ++dnl # The .mount callback has been removed, requiring all mount work ++dnl # to be done through the "new" mount API introduced in 5.2. ++dnl # + AC_DEFUN([ZFS_AC_KERNEL_SRC_FST_MOUNT], [ + ZFS_LINUX_TEST_SRC([file_system_type_mount], [ + #include +@@ -24,7 +28,8 @@ AC_DEFUN([ZFS_AC_KERNEL_FST_MOUNT], [ + AC_MSG_CHECKING([whether fst->mount() exists]) + ZFS_LINUX_TEST_RESULT([file_system_type_mount], [ + AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_FST_MOUNT, 1, [fst->mount() exists]) + ],[ +- ZFS_LINUX_TEST_ERROR([fst->mount()]) ++ AC_MSG_RESULT(no) + ]) + ]) +diff --git a/config/kernel-inode-state.m4 b/config/kernel-inode-state.m4 +new file mode 100644 +index 0000000..02ec138 +--- /dev/null ++++ b/config/kernel-inode-state.m4 +@@ -0,0 +1,23 @@ ++dnl # ++dnl # 6.19 API change. inode->i_state no longer accessible directly; helper ++dnl # functions exist. ++dnl # ++AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_STATE_READ_ONCE], [ ++ ZFS_LINUX_TEST_SRC([inode_state_read_once], [ ++ #include ++ ], [ ++ struct inode i = {}; ++ inode_state_read_once(&i); ++ ],[]) ++]) ++ ++AC_DEFUN([ZFS_AC_KERNEL_INODE_STATE_READ_ONCE], [ ++ AC_MSG_CHECKING([whether inode_state_read_once() exists]) ++ ZFS_LINUX_TEST_RESULT([inode_state_read_once], [ ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_INODE_STATE_READ_ONCE, 1, ++ [inode_state_read_once() exists]) ++ ],[ ++ AC_MSG_RESULT(no) ++ ]) ++]) +diff --git a/config/kernel.m4 b/config/kernel.m4 +index f095c8f..3eaba75 100644 +--- a/config/kernel.m4 ++++ b/config/kernel.m4 +@@ -59,6 +59,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ + ZFS_AC_KERNEL_SRC_ACL + ZFS_AC_KERNEL_SRC_INODE_SETATTR + ZFS_AC_KERNEL_SRC_INODE_GETATTR ++ ZFS_AC_KERNEL_SRC_INODE_STATE_READ_ONCE + ZFS_AC_KERNEL_SRC_SHOW_OPTIONS + ZFS_AC_KERNEL_SRC_SHRINKER + ZFS_AC_KERNEL_SRC_MKDIR +@@ -71,6 +72,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ + ZFS_AC_KERNEL_SRC_SETATTR_PREPARE + ZFS_AC_KERNEL_SRC_INSERT_INODE_LOCKED + ZFS_AC_KERNEL_SRC_DENTRY ++ ZFS_AC_KERNEL_SRC_DENTRY_ALIAS_D_U + ZFS_AC_KERNEL_SRC_TRUNCATE_SETSIZE + ZFS_AC_KERNEL_SRC_SECURITY_INODE + ZFS_AC_KERNEL_SRC_FST_MOUNT +@@ -117,7 +119,6 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ + ZFS_AC_KERNEL_SRC_ADD_DISK + ZFS_AC_KERNEL_SRC_KTHREAD + ZFS_AC_KERNEL_SRC_ZERO_PAGE +- ZFS_AC_KERNEL_SRC___COPY_FROM_USER_INATOMIC + ZFS_AC_KERNEL_SRC_IDMAP_MNT_API + ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS + ZFS_AC_KERNEL_SRC_IATTR_VFSID +@@ -139,6 +140,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ + ZFS_AC_KERNEL_SRC_NAMESPACE + ZFS_AC_KERNEL_SRC_INODE_GENERIC_DROP + ZFS_AC_KERNEL_SRC_KASAN_ENABLED ++ ZFS_AC_KERNEL_SRC_FILELOCK_HEADER + case "$host_cpu" in + powerpc*) + ZFS_AC_KERNEL_SRC_CPU_HAS_FEATURE +@@ -181,6 +183,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ + ZFS_AC_KERNEL_ACL + ZFS_AC_KERNEL_INODE_SETATTR + ZFS_AC_KERNEL_INODE_GETATTR ++ ZFS_AC_KERNEL_INODE_STATE_READ_ONCE + ZFS_AC_KERNEL_SHOW_OPTIONS + ZFS_AC_KERNEL_SHRINKER + ZFS_AC_KERNEL_MKDIR +@@ -193,6 +196,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ + ZFS_AC_KERNEL_SETATTR_PREPARE + ZFS_AC_KERNEL_INSERT_INODE_LOCKED + ZFS_AC_KERNEL_DENTRY ++ ZFS_AC_KERNEL_DENTRY_ALIAS_D_U + ZFS_AC_KERNEL_TRUNCATE_SETSIZE + ZFS_AC_KERNEL_SECURITY_INODE + ZFS_AC_KERNEL_FST_MOUNT +@@ -239,7 +243,6 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ + ZFS_AC_KERNEL_ADD_DISK + ZFS_AC_KERNEL_KTHREAD + ZFS_AC_KERNEL_ZERO_PAGE +- ZFS_AC_KERNEL___COPY_FROM_USER_INATOMIC + ZFS_AC_KERNEL_IDMAP_MNT_API + ZFS_AC_KERNEL_IDMAP_NO_USERNS + ZFS_AC_KERNEL_IATTR_VFSID +@@ -262,6 +265,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ + ZFS_AC_KERNEL_NAMESPACE + ZFS_AC_KERNEL_INODE_GENERIC_DROP + ZFS_AC_KERNEL_KASAN_ENABLED ++ ZFS_AC_KERNEL_FILELOCK_HEADER + case "$host_cpu" in + powerpc*) + ZFS_AC_KERNEL_CPU_HAS_FEATURE +diff --git a/include/os/linux/kernel/linux/dcache_compat.h b/include/os/linux/kernel/linux/dcache_compat.h +index 152e5a6..1fa6bf2 100644 +--- a/include/os/linux/kernel/linux/dcache_compat.h ++++ b/include/os/linux/kernel/linux/dcache_compat.h +@@ -32,7 +32,9 @@ + #define dname(dentry) ((char *)((dentry)->d_name.name)) + #define dlen(dentry) ((int)((dentry)->d_name.len)) + ++#ifdef HAVE_DENTRY_D_U_ALIASES + #define d_alias d_u.d_alias ++#endif + + /* + * Starting from Linux 5.13, flush_dcache_page() becomes an inline function +diff --git a/include/os/linux/kernel/linux/vfs_compat.h b/include/os/linux/kernel/linux/vfs_compat.h +index d9dc904..2421963 100644 +--- a/include/os/linux/kernel/linux/vfs_compat.h ++++ b/include/os/linux/kernel/linux/vfs_compat.h +@@ -269,4 +269,12 @@ zpl_is_32bit_api(void) + #define generic_drop_inode(ip) inode_generic_drop(ip) + #endif + ++#ifndef HAVE_INODE_STATE_READ_ONCE ++/* ++ * 6.19 API change. We should no longer access i_state directly. If the new ++ * helper function doesn't exist, define our own. ++ */ ++#define inode_state_read_once(ip) READ_ONCE(ip->i_state) ++#endif ++ + #endif /* _ZFS_VFS_H */ +diff --git a/include/os/linux/kernel/linux/xattr_compat.h b/include/os/linux/kernel/linux/xattr_compat.h +index f2f7e1e..39645c1 100644 +--- a/include/os/linux/kernel/linux/xattr_compat.h ++++ b/include/os/linux/kernel/linux/xattr_compat.h +@@ -130,10 +130,27 @@ zpl_acl_from_xattr(const void *value, int size) + return (posix_acl_from_xattr(kcred->user_ns, value, size)); + } + ++/* ++ * Linux 7.0 API change. posix_acl_to_xattr() changed from filling the ++ * caller-provided buffer to allocating a buffer with enough space and ++ * returning it. We wrap this up by copying the result into the provided ++ * buffer and freeing the allocated buffer. ++ */ + static inline int + zpl_acl_to_xattr(struct posix_acl *acl, void *value, int size) + { ++#ifdef HAVE_POSIX_ACL_TO_XATTR_ALLOC ++ size_t s = 0; ++ void *v = posix_acl_to_xattr(kcred->user_ns, acl, &s, ++ kmem_flags_convert(KM_SLEEP)); ++ if (v == NULL) ++ return (-ENOMEM); ++ memcpy(value, v, MIN(size, s)); ++ kfree(v); ++ return (0); ++#else + return (posix_acl_to_xattr(kcred->user_ns, acl, value, size)); ++#endif + } + + #endif /* _ZFS_XATTR_H */ +diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c +index 1bd3500..ab7457b 100644 +--- a/module/os/linux/zfs/vdev_disk.c ++++ b/module/os/linux/zfs/vdev_disk.c +@@ -445,7 +445,11 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize, + v->vdev_has_securetrim = bdev_secure_discard_supported(bdev); + + /* Inform the ZIO pipeline that we are non-rotational */ ++#ifdef HAVE_BLK_QUEUE_ROT ++ v->vdev_nonrot = !blk_queue_rot(bdev_get_queue(bdev)); ++#else + v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev)); ++#endif + + /* Physical volume size in bytes for the partition */ + *psize = bdev_capacity(bdev); +diff --git a/module/os/linux/zfs/zfs_vnops_os.c b/module/os/linux/zfs/zfs_vnops_os.c +index a788e3f..da09fab 100644 +--- a/module/os/linux/zfs/zfs_vnops_os.c ++++ b/module/os/linux/zfs/zfs_vnops_os.c +@@ -3513,7 +3513,8 @@ zfs_link(znode_t *tdzp, znode_t *szp, char *name, cred_t *cr, + boolean_t is_tmpfile = 0; + uint64_t txg; + +- is_tmpfile = (sip->i_nlink == 0 && (sip->i_state & I_LINKABLE)); ++ is_tmpfile = (sip->i_nlink == 0 && ++ (inode_state_read_once(sip) & I_LINKABLE)); + + ASSERT(S_ISDIR(ZTOI(tdzp)->i_mode)); + +diff --git a/module/os/linux/zfs/zpl_file.c b/module/os/linux/zfs/zpl_file.c +index f7691c0..efcb400 100644 +--- a/module/os/linux/zfs/zpl_file.c ++++ b/module/os/linux/zfs/zpl_file.c +@@ -43,6 +43,9 @@ + #ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO + #include + #endif ++#ifdef HAVE_FILELOCK_HEADER ++#include ++#endif + + /* + * When using fallocate(2) to preallocate space, inflate the requested +@@ -1242,6 +1245,7 @@ const struct file_operations zpl_file_operations = { + .mmap = zpl_mmap, + .fsync = zpl_fsync, + .fallocate = zpl_fallocate, ++ .setlease = generic_setlease, + .copy_file_range = zpl_copy_file_range, + #ifdef HAVE_VFS_CLONE_FILE_RANGE + .clone_file_range = zpl_clone_file_range, +@@ -1264,6 +1268,7 @@ const struct file_operations zpl_dir_file_operations = { + .read = generic_read_dir, + .iterate_shared = zpl_iterate, + .fsync = zpl_fsync, ++ .setlease = generic_setlease, + .unlocked_ioctl = zpl_ioctl, + #ifdef CONFIG_COMPAT + .compat_ioctl = zpl_compat_ioctl, +diff --git a/module/os/linux/zfs/zpl_super.c b/module/os/linux/zfs/zpl_super.c +index 347b352..5baa4f6 100644 +--- a/module/os/linux/zfs/zpl_super.c ++++ b/module/os/linux/zfs/zpl_super.c +@@ -24,6 +24,7 @@ + * Copyright (c) 2023, Datto Inc. All rights reserved. + * Copyright (c) 2025, Klara, Inc. + * Copyright (c) 2025, Rob Norris ++ * Copyright (c) 2026, TrueNAS. + */ + + +@@ -36,6 +37,10 @@ + #include + #include + ++#ifndef HAVE_FST_MOUNT ++#include ++#endif ++ + /* + * What to do when the last reference to an inode is released. If 0, the kernel + * will cache it on the superblock. If 1, the inode will be freed immediately. +@@ -504,6 +509,74 @@ zpl_prune_sb(uint64_t nr_to_scan, void *arg) + #endif + } + ++#ifndef HAVE_FST_MOUNT ++/* ++ * In kernel 7.0, the file_system_type->mount() and ++ * super_operations->remount_fs() callbacks have been removed, requiring all ++ * users to convert to the "new" fs_context-based mount API introduced in 5.2. ++ * ++ * This is the simplest compatibility shim possible to adapt the fs_context ++ * interface to the old-style calls. Although this interface exists in almost ++ * all versions of Linux currently supported by OpenZFS, we only use it when ++ * the kernel-provided shims are unavailable, to avoid bugs in these new shims ++ * affecting all OpenZFS deployments. ++ */ ++static int ++zpl_parse_monolithic(struct fs_context *fc, void *data) ++{ ++ if (data == NULL) ++ return (0); ++ ++ /* ++ * Because we supply a .parse_monolithic callback, the kernel does ++ * no consideration of the options blob at all. Because of this, we ++ * have to give LSMs a first look at it. They will remove any options ++ * of interest to them (eg the SELinux *context= options). ++ */ ++ int err = security_sb_eat_lsm_opts((char *)data, &fc->security); ++ if (err) ++ return (err); ++ ++ /* ++ * Whatever is left we stash on in the fs_context so we can pass it ++ * down to zfs_domount() or zfs_remount() later. ++ */ ++ fc->fs_private = data; ++ return (0); ++} ++ ++static int ++zpl_get_tree(struct fs_context *fc) ++{ ++ struct dentry *root = ++ zpl_mount(fc->fs_type, fc->sb_flags, fc->source, fc->fs_private); ++ if (IS_ERR(root)) ++ return (PTR_ERR(root)); ++ ++ fc->root = root; ++ return (0); ++} ++ ++static int ++zpl_reconfigure(struct fs_context *fc) ++{ ++ return (zpl_remount_fs(fc->root->d_sb, &fc->sb_flags, fc->fs_private)); ++} ++ ++const struct fs_context_operations zpl_fs_context_operations = { ++ .parse_monolithic = zpl_parse_monolithic, ++ .get_tree = zpl_get_tree, ++ .reconfigure = zpl_reconfigure, ++}; ++ ++static int ++zpl_init_fs_context(struct fs_context *fc) ++{ ++ fc->ops = &zpl_fs_context_operations; ++ return (0); ++} ++#endif ++ + const struct super_operations zpl_super_operations = { + .alloc_inode = zpl_inode_alloc, + #ifdef HAVE_SOPS_FREE_INODE +@@ -517,7 +590,9 @@ const struct super_operations zpl_super_operations = { + .put_super = zpl_put_super, + .sync_fs = zpl_sync_fs, + .statfs = zpl_statfs, ++#ifdef HAVE_FST_MOUNT + .remount_fs = zpl_remount_fs, ++#endif + .show_devname = zpl_show_devname, + .show_options = zpl_show_options, + .show_stats = NULL, +@@ -560,7 +635,11 @@ struct file_system_type zpl_fs_type = { + #else + .fs_flags = FS_USERNS_MOUNT, + #endif ++#ifdef HAVE_FST_MOUNT + .mount = zpl_mount, ++#else ++ .init_fs_context = zpl_init_fs_context, ++#endif + .kill_sb = zpl_kill_sb, + }; + +diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh +index 09a15ba..697c3f3 100755 +--- a/scripts/zfs-tests.sh ++++ b/scripts/zfs-tests.sh +@@ -294,6 +294,16 @@ constrain_path() { + SYSTEM_DIRS="/usr/local/bin /usr/local/sbin" + SYSTEM_DIRS="$SYSTEM_DIRS /usr/bin /usr/sbin /bin /sbin $LIBEXEC_DIR" + ++ SYSTEM_FILES="$SYSTEM_FILES_COMMON" ++ ZFSTEST_FILES="$ZFSTEST_FILES_COMMON" ++ if [ "$UNAME" = "FreeBSD" ] ; then ++ SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD" ++ ZFSTEST_FILES="$ZFSTEST_FILES $ZFSTEST_FILES_FREEBSD" ++ else ++ SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX" ++ ZFSTEST_FILES="$ZFSTEST_FILES $ZFSTEST_FILES_LINUX" ++ fi ++ + if [ "$INTREE" = "yes" ]; then + # Constrained path set to $(top_builddir)/tests/zfs-tests/bin + STF_PATH="$BIN_DIR" +@@ -326,12 +336,6 @@ constrain_path() { + fi + + # Standard system utilities +- SYSTEM_FILES="$SYSTEM_FILES_COMMON" +- if [ "$UNAME" = "FreeBSD" ] ; then +- SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_FREEBSD" +- else +- SYSTEM_FILES="$SYSTEM_FILES $SYSTEM_FILES_LINUX" +- fi + create_links "$SYSTEM_DIRS" "$SYSTEM_FILES" + + # Exceptions +diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run +index 339361c..1e51dff 100644 +--- a/tests/runfiles/linux.run ++++ b/tests/runfiles/linux.run +@@ -141,6 +141,10 @@ pre = + post = + tags = ['functional', 'largest_pool'] + ++[tests/functional/lease:Linux] ++tests = ['lease_setlease'] ++tags = ['functional', 'lease'] ++ + [tests/functional/longname:Linux] + tests = ['longname_001_pos', 'longname_002_pos', 'longname_003_pos'] + tags = ['functional', 'longname'] +diff --git a/tests/zfs-tests/cmd/.gitignore b/tests/zfs-tests/cmd/.gitignore +index 62f1684..335e4ce 100644 +--- a/tests/zfs-tests/cmd/.gitignore ++++ b/tests/zfs-tests/cmd/.gitignore +@@ -58,3 +58,4 @@ + /sha2_test + /idmap_util + /statx ++/setlease +diff --git a/tests/zfs-tests/cmd/Makefile.am b/tests/zfs-tests/cmd/Makefile.am +index 85c3cf3..f07e7fd 100644 +--- a/tests/zfs-tests/cmd/Makefile.am ++++ b/tests/zfs-tests/cmd/Makefile.am +@@ -132,6 +132,7 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/statx + scripts_zfs_tests_bin_PROGRAMS += %D%/xattrtest + scripts_zfs_tests_bin_PROGRAMS += %D%/zed_fd_spill-zedlet + scripts_zfs_tests_bin_PROGRAMS += %D%/idmap_util ++scripts_zfs_tests_bin_PROGRAMS += %D%/setlease + + %C%_idmap_util_LDADD = libspl.la + +diff --git a/tests/zfs-tests/cmd/setlease.c b/tests/zfs-tests/cmd/setlease.c +new file mode 100644 +index 0000000..12bcbd9 +--- /dev/null ++++ b/tests/zfs-tests/cmd/setlease.c +@@ -0,0 +1,126 @@ ++// SPDX-License-Identifier: CDDL-1.0 ++/* ++ * CDDL HEADER START ++ * ++ * The contents of this file are subject to the terms of the ++ * Common Development and Distribution License (the "License"). ++ * You may not use this file except in compliance with the License. ++ * ++ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE ++ * or https://opensource.org/licenses/CDDL-1.0. ++ * See the License for the specific language governing permissions ++ * and limitations under the License. ++ * ++ * When distributing Covered Code, include this CDDL HEADER in each ++ * file and include the License file at usr/src/OPENSOLARIS.LICENSE. ++ * If applicable, add the following below this CDDL HEADER, with the ++ * fields enclosed by brackets "[]" replaced with your own identifying ++ * information: Portions Copyright [yyyy] [name of copyright owner] ++ * ++ * CDDL HEADER END ++ */ ++ ++/* ++ * Copyright (c) 2026, TrueNAS. ++ */ ++ ++/* ++ * This is a sanity check test for the F_SETLEASE and F_GETLEASE fcntl() calls. ++ * We use the generic kernel implementation, but we want to be alerted if it ++ * ever breaks. ++ * ++ * This is not a comprehensive test. It would be nice if it could be! ++ */ ++ ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++static int ++get_lease(int fd) { ++ int r = fcntl(fd, F_GETLEASE); ++ if (r < 0) { ++ perror("fcntl(GETLEASE)"); ++ exit(2); ++ } ++ return (r); ++} ++ ++static int ++set_lease(int fd, int lease) { ++ return (fcntl(fd, F_SETLEASE, lease) < 0 ? errno : 0); ++} ++ ++static const char *lease_str[] = { ++ [F_RDLCK] = "RDLCK", ++ [F_WRLCK] = "WRLCK", ++ [F_UNLCK] = "UNLCK", ++}; ++ ++static void ++assert_lease(int fd, int expect) { ++ int got = get_lease(fd); ++ if (got != expect) { ++ fprintf(stderr, "ASSERT_LEASE: expected %s [%d], got %s [%d]\n", ++ lease_str[expect], expect, lease_str[got], got); ++ abort(); ++ } ++ printf("ok: lease is %s\n", lease_str[got]); ++} ++ ++static void ++assert_set_lease(int fd, int lease) { ++ int err = set_lease(fd, lease); ++ if (err != 0) { ++ fprintf(stderr, "ASSERT_SET_LEASE: tried %s [%d], error: %s\n", ++ lease_str[lease], lease, strerror(err)); ++ abort(); ++ } ++ printf("ok: set lease to %s\n", lease_str[lease]); ++} ++ ++int ++main(int argc, char **argv) ++{ ++ if (argc != 2) { ++ fprintf(stderr, "usage: %s \n", argv[0]); ++ exit(1); ++ } ++ ++ /* create and open file, read+write */ ++ int fd = open(argv[1], O_CREAT|O_RDONLY, S_IRWXU|S_IRWXG|S_IRWXO); ++ if (fd < 0) { ++ perror("open"); ++ exit(2); ++ } ++ printf("ok: opened file RDONLY\n"); ++ ++ /* fd starts with no lease */ ++ assert_lease(fd, F_UNLCK); ++ ++ /* fd is readonly, so can take read lease */ ++ assert_set_lease(fd, F_RDLCK); ++ /* confirm read lease */ ++ assert_lease(fd, F_RDLCK); ++ ++ /* no other openers, so can take write lease */ ++ assert_set_lease(fd, F_WRLCK); ++ /* confirm write lease */ ++ assert_lease(fd, F_WRLCK); ++ ++ /* release lease */ ++ assert_set_lease(fd, F_UNLCK); ++ /* confirm lease released */ ++ assert_lease(fd, F_UNLCK); ++ ++ close(fd); ++ ++ return (0); ++} +diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg +index 1c4d25e..4ba9aa7 100644 +--- a/tests/zfs-tests/include/commands.cfg ++++ b/tests/zfs-tests/include/commands.cfg +@@ -182,7 +182,7 @@ export ZFS_FILES='zdb + zfs_ids_to_path + zpool_influxdb' + +-export ZFSTEST_FILES='badsend ++export ZFSTEST_FILES_COMMON='badsend + btree_test + chg_usr_exec + clonefile +@@ -241,3 +241,6 @@ export ZFSTEST_FILES='badsend + zfs_diff-socket + dosmode_readonly_write + idmap_util' ++ ++export ZFSTEST_FILES_LINUX=' ++ setlease' +diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am +index a4b3e73..a52532d 100644 +--- a/tests/zfs-tests/tests/Makefile.am ++++ b/tests/zfs-tests/tests/Makefile.am +@@ -1653,6 +1653,9 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ + functional/io/psync.ksh \ + functional/io/setup.ksh \ + functional/io/sync.ksh \ ++ functional/lease/cleanup.ksh \ ++ functional/lease/lease_setlease.ksh \ ++ functional/lease/setup.ksh \ + functional/l2arc/cleanup.ksh \ + functional/l2arc/l2arc_arcstats_pos.ksh \ + functional/l2arc/l2arc_l2miss_pos.ksh \ +diff --git a/tests/zfs-tests/tests/functional/lease/cleanup.ksh b/tests/zfs-tests/tests/functional/lease/cleanup.ksh +new file mode 100755 +index 0000000..5e73dd3 +--- /dev/null ++++ b/tests/zfs-tests/tests/functional/lease/cleanup.ksh +@@ -0,0 +1,26 @@ ++#!/bin/ksh -p ++# SPDX-License-Identifier: CDDL-1.0 ++# ++# CDDL HEADER START ++# ++# The contents of this file are subject to the terms of the ++# Common Development and Distribution License (the "License"). ++# You may not use this file except in compliance with the License. ++# ++# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE ++# or https://opensource.org/licenses/CDDL-1.0. ++# See the License for the specific language governing permissions ++# and limitations under the License. ++# ++# When distributing Covered Code, include this CDDL HEADER in each ++# file and include the License file at usr/src/OPENSOLARIS.LICENSE. ++# If applicable, add the following below this CDDL HEADER, with the ++# fields enclosed by brackets "[]" replaced with your own identifying ++# information: Portions Copyright [yyyy] [name of copyright owner] ++# ++# CDDL HEADER END ++# ++ ++. $STF_SUITE/include/libtest.shlib ++ ++default_cleanup +diff --git a/tests/zfs-tests/tests/functional/lease/lease_setlease.ksh b/tests/zfs-tests/tests/functional/lease/lease_setlease.ksh +new file mode 100755 +index 0000000..8647d01 +--- /dev/null ++++ b/tests/zfs-tests/tests/functional/lease/lease_setlease.ksh +@@ -0,0 +1,44 @@ ++#!/bin/ksh -p ++# SPDX-License-Identifier: CDDL-1.0 ++# ++# CDDL HEADER START ++# ++# The contents of this file are subject to the terms of the ++# Common Development and Distribution License (the "License"). ++# You may not use this file except in compliance with the License. ++# ++# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE ++# or https://opensource.org/licenses/CDDL-1.0. ++# See the License for the specific language governing permissions ++# and limitations under the License. ++# ++# When distributing Covered Code, include this CDDL HEADER in each ++# file and include the License file at usr/src/OPENSOLARIS.LICENSE. ++# If applicable, add the following below this CDDL HEADER, with the ++# fields enclosed by brackets "[]" replaced with your own identifying ++# information: Portions Copyright [yyyy] [name of copyright owner] ++# ++# CDDL HEADER END ++# ++ ++# ++# Copyright (c) 2026, TrueNAS. ++# ++ ++. $STF_SUITE/include/libtest.shlib ++ ++verify_runnable "both" ++ ++leasefile=/$TESTPOOL/leasefile ++ ++function cleanup ++{ ++ rm -f $leasefile ++} ++ ++log_assert "F_SETLEASE is supported" ++log_onexit cleanup ++ ++log_must setlease $leasefile ++ ++log_pass "F_SETLEASE is supported" +diff --git a/tests/zfs-tests/tests/functional/lease/setup.ksh b/tests/zfs-tests/tests/functional/lease/setup.ksh +new file mode 100755 +index 0000000..09da91b +--- /dev/null ++++ b/tests/zfs-tests/tests/functional/lease/setup.ksh +@@ -0,0 +1,27 @@ ++#!/bin/ksh -p ++# SPDX-License-Identifier: CDDL-1.0 ++# ++# CDDL HEADER START ++# ++# The contents of this file are subject to the terms of the ++# Common Development and Distribution License (the "License"). ++# You may not use this file except in compliance with the License. ++# ++# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE ++# or https://opensource.org/licenses/CDDL-1.0. ++# See the License for the specific language governing permissions ++# and limitations under the License. ++# ++# When distributing Covered Code, include this CDDL HEADER in each ++# file and include the License file at usr/src/OPENSOLARIS.LICENSE. ++# If applicable, add the following below this CDDL HEADER, with the ++# fields enclosed by brackets "[]" replaced with your own identifying ++# information: Portions Copyright [yyyy] [name of copyright owner] ++# ++# CDDL HEADER END ++# ++ ++. $STF_SUITE/include/libtest.shlib ++ ++DISK=${DISKS%% *} ++default_setup $DISK diff --git a/os/mkosi/components/zfs/zfs-build.sh b/os/mkosi/components/zfs/zfs-build.sh new file mode 100755 index 000000000..bdfe53b17 --- /dev/null +++ b/os/mkosi/components/zfs/zfs-build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +B=$(realpath -m "${1:?build directory required}") +KERNEL_SRC=$(realpath "${2:?kernel source required}") +KERNEL_BUILD=$(realpath "${3:?kernel build directory required}") +ROOT_STAGE=$(realpath -m "${4:?rootfs staging tree required}") +KERNEL_STAGE=$(realpath -m "${5:?kernel staging tree required}") +src="$B/zfs" +export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$B=/usr/src/zfs" +export CXXFLAGS="${CXXFLAGS:-} -O2 -g0 -ffile-prefix-map=$B=/usr/src/zfs" +export KCFLAGS="${KCFLAGS:-} -fdebug-prefix-map=$B=/usr/src/zfs -fmacro-prefix-map=$B=/usr/src/zfs -fdebug-prefix-map=$KERNEL_SRC=/usr/src/linux -fmacro-prefix-map=$KERNEL_SRC=/usr/src/linux -fdebug-prefix-map=$KERNEL_BUILD=/usr/src/linux-build -fmacro-prefix-map=$KERNEL_BUILD=/usr/src/linux-build" +export KAFLAGS="${KAFLAGS:-} $KCFLAGS" +export KCPPFLAGS="${KCPPFLAGS:-} $KCFLAGS" +if [[ ! -d $src/.git ]]; then git init -q "$src"; git -C "$src" remote add origin https://github.com/openzfs/zfs.git; fi +git -C "$src" fetch -q --depth=1 origin "$ZFS_REVISION" +git -C "$src" checkout -q --detach FETCH_HEAD +git -C "$src" reset -q --hard "$ZFS_REVISION" +git -C "$src" clean -qfdx +patch -d "$src" -p1 --fuzz=0 < "$SELF/components/zfs/patches/0001-linux-6.19-7.1-compat.patch" +patch -d "$src" -p1 --fuzz=0 < "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch" +(cd "$src" && ./autogen.sh) +mkdir -p "$B/build" +(cd "$B/build" && "$src/configure" --prefix=/usr --with-linux="$KERNEL_SRC" \ + --with-linux-obj="$KERNEL_BUILD" --with-config=all --enable-systemd \ + --disable-pyzfs --without-dracutdir --with-udevdir=/usr/lib/udev \ + --with-systemdunitdir=/usr/lib/systemd/system) +# Linux 7.1 drives pahole 1.25's nondeterministic optimized encoder for +# external modules. Kernel/in-tree BTF remains enabled; omit only ZFS module +# BTF, as is also required for NVIDIA's external modules on this toolchain. +make -C "$B/build" -j"${JOBS:-$(nproc)}" CONFIG_DEBUG_INFO_BTF_MODULES= +make -C "$B/build" CONFIG_DEBUG_INFO_BTF_MODULES= DESTDIR="$ROOT_STAGE" install +# ZFS installs kernel modules below DESTDIR/lib; merge them into usr-merge. +if [[ -d $ROOT_STAGE/lib/modules ]]; then + mkdir -p "$KERNEL_STAGE/usr/lib/modules" + cp -a "$ROOT_STAGE/lib/modules/." "$KERNEL_STAGE/usr/lib/modules/" + rm -rf "${ROOT_STAGE:?}/lib/modules" +fi +rm -rf "$ROOT_STAGE/usr/share/zfs" "$ROOT_STAGE/usr/share/initramfs-tools" +ln -sfn usr/lib "$KERNEL_STAGE/lib" +depmod -b "$KERNEL_STAGE" "$KERNEL_VERSION-dstack" +rm -f "$KERNEL_STAGE/lib" +find "$ROOT_STAGE" "$KERNEL_STAGE" -print0 | xargs -0r touch -h -d "@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/components/zfs/zfs.sh b/os/mkosi/components/zfs/zfs.sh new file mode 100644 index 000000000..60253c6f1 --- /dev/null +++ b/os/mkosi/components/zfs/zfs.sh @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash disable=SC2034 +COMPONENT_NAME=zfs +COMPONENT_CACHE_PATHS=(component-stages/zfs-root component-stages/zfs-kernel) +COMPONENT_ROOTFS_TREES=(component-stages/zfs-root) +COMPONENT_KERNEL_TREES=(component-stages/zfs-kernel) + +component_cache_key() { + key_dependency kernel + key_value "$ZFS_VERSION" "$ZFS_REVISION" + key_file "$COMPONENT_PATH/zfs-build.sh" \ + "$COMPONENT_PATH/patches/0001-linux-6.19-7.1-compat.patch" \ + "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch" + key_tools gcc make autoconf automake + key_packages libssl-dev libelf-dev zlib1g-dev libtool uuid-dev +} + +component_build() { + "$COMPONENT_PATH/zfs-build.sh" "$WORK/zfs-build" \ + "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" \ + "$WORK/component-stages/zfs-root" "$WORK/component-stages/zfs-kernel" +} + +component_prepare_outputs() { + find "$WORK/component-stages/zfs-kernel/usr/lib/modules" -maxdepth 2 \ + -type f -name 'modules.*' -delete +} diff --git a/os/mkosi/mkosi.build b/os/mkosi/mkosi.build new file mode 100755 index 000000000..2d1a76b38 --- /dev/null +++ b/os/mkosi/mkosi.build @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +# Enter the mkosi build root so every compiler and build dependency comes from +# the pinned Debian snapshot rather than from the host. +if [[ ${container:-} != mkosi ]]; then + exec mkosi-chroot "$CHROOT_SCRIPT" "$@" +fi + +ROOT="$SRCDIR/dstack" +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" + +export HOME=/var/tmp/dstack-home +export CARGO_HOME=/var/tmp/dstack-cargo-home +export GOCACHE=/var/tmp/dstack-go-cache +export GOMODCACHE=/var/tmp/dstack-go-mod-cache +export XDG_CACHE_HOME=/var/tmp/dstack-xdg-cache +export TAR_OPTIONS="${TAR_OPTIONS:-} --no-same-owner" +mkdir -p "$HOME" "$CARGO_HOME" "$GOCACHE" "$GOMODCACHE" "$XDG_CACHE_HOME" + +cache_enabled=${DSTACK_COMPONENT_CACHE:-0} +if [[ $cache_enabled == 1 ]]; then + export DSTACK_DEV_CACHE_DIR="$SRCDIR/component-cache/components" + toolchain_downloads="$SRCDIR/component-cache/toolchains" +else + toolchain_downloads=/var/tmp/dstack-toolchain-downloads +fi +export PATH="/opt/dstack-toolchains/rust/bin:/opt/dstack-toolchains/go/bin:$PATH" +"$SELF/scripts/install-toolchains.sh" "$toolchain_downloads" +printf 'gcc: '; gcc --version | head -1 +printf 'g++: '; g++ --version | head -1 +printf 'ld: '; ld --version | head -1 +printf 'cmake: '; cmake --version | head -1 +dpkg-query -W -f='build-package: ${binary:Package}=${Version}\n' \ + gcc g++ binutils cmake make ninja-build dwarves + +work=/var/tmp/dstack-component-work +kernel_stage=/var/tmp/dstack-kernel-stage +mkdir -p "$work" "$kernel_stage" +args=() +[[ $cache_enabled == 1 ]] && args+=(--dev-cache) +"$SELF/scripts/build-components.sh" "${args[@]}" \ + "$work" "$DESTDIR" "$kernel_stage" "${DSTACK_BUILD_FLAVOR:?}" + +# Normalize any legacy physical directories before mkosi copies DESTDIR over +# Debian's usr-merged root. Use the same strict merger so this cannot overwrite +# a component-owned path silently. +for legacy in bin sbin lib lib64; do + if [[ -d $DESTDIR/$legacy && ! -L $DESTDIR/$legacy ]]; then + mkdir -p "$DESTDIR/usr/$legacy" + "$SELF/scripts/merge-component-trees.py" "$DESTDIR/usr/$legacy" \ + "legacy-$legacy=$DESTDIR/$legacy" + rm -rf "${DESTDIR:?}/$legacy" + fi +done + +# Merge kernel/firmware outputs into the build install tree. mkosi copies +# DESTDIR into the final image after removing the build-package overlay. +"$SELF/scripts/merge-component-trees.py" "$DESTDIR" \ + "kernel=$kernel_stage" +ln -sfn usr/lib "$DESTDIR/lib" +depmod -b "$DESTDIR" "$KERNEL_VERSION-dstack" +rm -f "$DESTDIR/lib" +if [[ $DSTACK_BUILD_FLAVOR == prod ]]; then + find "$DESTDIR/usr/lib/modules" -type f -name '*.ko' \ + -exec objcopy --strip-debug {} \; +fi +find "$DESTDIR" -print0 | xargs -0r touch --no-dereference \ + --date="@${SOURCE_DATE_EPOCH:?}" diff --git a/os/mkosi/mkosi.clean b/os/mkosi/mkosi.clean new file mode 100755 index 000000000..379fc6069 --- /dev/null +++ b/os/mkosi/mkosi.clean @@ -0,0 +1,7 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +[[ -n ${OUTPUTDIR:-} ]] || exit 0 +find "$OUTPUTDIR" -maxdepth 1 \( -name 'dstack-*' -o -name '.dstack-*' \ + -o -name '.image-tools' \) -exec rm -rf -- {} + diff --git a/os/mkosi/mkosi.conf b/os/mkosi/mkosi.conf new file mode 100644 index 000000000..1f4e738a3 --- /dev/null +++ b/os/mkosi/mkosi.conf @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +[Config] +MinimumVersion=26 + +[Distribution] +Distribution=debian +Release=trixie +Architecture=x86-64 +Snapshot=20260721T000000Z +Repositories=main,contrib,non-free-firmware + +[Output] +Format=tar +# The native tar output is deliberately replaced by the Yocto-compatible tar +# in mkosi.postoutput. This keeps the complete mkosi lifecycle enabled without +# leaving an unrelated rootfs image beside the release artifacts. +Output=dstack-0.6.0 +CompressOutput=gz +SplitArtifacts= +ImageId=dstack +ImageVersion=0.6.0 + +[Content] +Bootable=no +Bootloader=none +Hostname=dstack +Packages= + systemd + systemd-boot + systemd-resolved + dbus + ca-certificates + bash + coreutils + iproute2 + iptables + nftables + docker.io + docker-cli + docker-compose + containerd + chrony + wireguard-tools + cryptsetup + dmsetup + tpm2-tools + curl + jq + fuse3 + pigz + xfsprogs + e2fsprogs + gdisk + parted + busybox-static + cryptsetup-bin + systemd-boot-efi + acpid + numactl +SkeletonTrees=mkosi.skeleton +PostInstallationScripts=mkosi.postinst +RemoveFiles= + /var/lib/containerd/.dstack-keep + /var/lib/docker/.dstack-keep + /var/lib/tpm2-tss/system/keystore/.dstack-keep + /var/mail/.dstack-keep + /var/volatile/.dstack-keep +KernelCommandLine=console=ttyS0,115200n8 console=tty0 panic=30 coherent_pool=64M +CleanPackageMetadata=yes +WithDocs=no +Locale=C.UTF-8 +Timezone=UTC +BuildPackages= + gcc + g++ + binutils + make + cmake + ninja-build + pkg-config + autoconf + automake + libtool + gettext + bc + bison + flex + dwarves + nasm + acpica-tools + uuid-dev + libssl-dev + libelf-dev + libtirpc-dev + libcurl4-openssl-dev + libxml2-dev + libxmlsec1-dev + zlib1g-dev + libudev-dev + libaio-dev + libattr1-dev + libblkid-dev + libseccomp-dev + libcap-dev + rpcsvc-proto + git + curl + ca-certificates + patch + xz-utils + bzip2 + zstd + tar + file + perl + python3 + gawk + kmod + coreutils + findutils + +[Build] +WithNetwork=yes +BuildSources=../..:dstack +BuildSourcesEphemeral=yes +ToolsTree=yes +ToolsTreeDistribution=debian +ToolsTreeRelease=trixie +# Image assembly needs mkosi's general-purpose tools plus the explicit list +# below, not virtualization runtimes or foreign distribution package managers. +ToolsTreeProfiles=misc +ToolsTreeRepositories=main +ToolsTreePackages= + cpio + cryptsetup-bin + dosfstools + gdisk + gzip + jq + mtools + pax-utils + squashfs-tools + systemd-ukify + tar +History=no + +[Validation] +SecureBoot=no diff --git a/os/mkosi/mkosi.finalize b/os/mkosi/mkosi.finalize new file mode 100755 index 000000000..5f78ec095 --- /dev/null +++ b/os/mkosi/mkosi.finalize @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +ROOT="$SRCDIR/dstack" +SELF="$ROOT/os/mkosi" +TREE=${BUILDROOT:?} +FLAVOR=${DSTACK_BUILD_FLAVOR:?} +WORK="$OUTPUTDIR/.dstack-$FLAVOR" +KSTAGE="$WORK/kernel-stage" +ARTIFACTS="$WORK/artifacts" + +"$SELF/scripts/normalize-skeleton-modes.sh" "$TREE" +mkdir -p "$KSTAGE/usr/lib" "$OUTPUTDIR/.image-tools" +cp -a "$TREE/usr/lib/modules" "$KSTAGE/usr/lib/" +install -m0644 "$TREE/usr/lib/dstack/firmware/ovmf.fd" "$KSTAGE/ovmf.fd" +install -m0644 "$TREE/usr/lib/dstack/firmware/ovmf-sev.fd" "$KSTAGE/ovmf-sev.fd" +install -m0755 "$TREE/usr/lib/dstack/image-tools/"* "$OUTPUTDIR/.image-tools/" +rm -rf "$TREE/usr/lib/dstack/firmware" "$TREE/usr/lib/dstack/image-tools" + +rm -f "$TREE/var/cache/ldconfig/aux-cache" +"$SELF/tests/check-parity.py" "$SELF/parity.json" "$TREE" "$KSTAGE" "$FLAVOR" +"$SELF/scripts/make-release-artifacts.sh" "$TREE" "$KSTAGE" "$ARTIFACTS" "$FLAVOR" diff --git a/os/mkosi/mkosi.postinst b/os/mkosi/mkosi.postinst new file mode 100755 index 000000000..dbe4a38b3 --- /dev/null +++ b/os/mkosi/mkosi.postinst @@ -0,0 +1,7 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +B=${BUILDROOT:?} +rm -f "$B/etc/machine-id" "$B/var/lib/dbus/machine-id" +: > "$B/etc/machine-id" diff --git a/os/mkosi/mkosi.postoutput b/os/mkosi/mkosi.postoutput new file mode 100755 index 000000000..78d3bcbc8 --- /dev/null +++ b/os/mkosi/mkosi.postoutput @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +ROOT="$SRCDIR/dstack" +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +FLAVOR=${DSTACK_BUILD_FLAVOR:?} +WORK="$OUTPUTDIR/.dstack-$FLAVOR" + +# mkosi creates an extensionless convenience symlink for its compressed tar. +# The release directory has that same Yocto-defined name, so remove the link +# before assemble.sh atomically replaces the native tar and creates the tree. +rm -f "$OUTPUTDIR/dstack-$DSTACK_VERSION" +DSTACK_MR_BIN="$OUTPUTDIR/.image-tools/dstack-mr" \ +NITRO_TPM_PCR_COMPUTE_BIN="$OUTPUTDIR/.image-tools/nitro-tpm-pcr-compute" \ +DIST_DIR="$OUTPUTDIR" \ +TAR_OPTIONS="--sort=name --mtime=@$SOURCE_DATE_EPOCH --owner=0 --group=0 --numeric-owner" \ + "$ROOT/os/image/assemble.sh" --manifest "$WORK/artifacts/artifact-manifest.json" + +release_name=dstack +[[ $FLAVOR == dev ]] && release_name=dstack-dev +"$SELF/tests/check-output.sh" "$OUTPUTDIR/$release_name-$DSTACK_VERSION" "$FLAVOR" +rm -rf "$WORK" "$OUTPUTDIR/.image-tools" diff --git a/os/mkosi/mkosi.profiles/dev/mkosi.conf b/os/mkosi/mkosi.profiles/dev/mkosi.conf new file mode 100644 index 000000000..3a01aa62a --- /dev/null +++ b/os/mkosi/mkosi.profiles/dev/mkosi.conf @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +[Content] +Packages= + gdb + openssh-server + strace + tcpdump + vim + +[Build] +Environment=DSTACK_BUILD_FLAVOR=dev diff --git a/os/mkosi/mkosi.profiles/prod/mkosi.conf b/os/mkosi/mkosi.profiles/prod/mkosi.conf new file mode 100644 index 000000000..af6b57947 --- /dev/null +++ b/os/mkosi/mkosi.profiles/prod/mkosi.conf @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +[Content] +RemoveFiles= + /usr/include + /usr/share/OVMF + /usr/share/ovmf + /usr/share/doc + /usr/share/info + /usr/share/locale + /usr/share/man + /usr/**/*.a + /usr/**/*.la + /var/cache/debconf + /var/lib/dbus/machine-id + /var/lib/dpkg + /var/lib/ucf + /var/log/* + +[Build] +Environment=DSTACK_BUILD_FLAVOR=prod diff --git a/os/mkosi/mkosi.skeleton/etc/apt/apt.conf.d/99dstack-reproducible b/os/mkosi/mkosi.skeleton/etc/apt/apt.conf.d/99dstack-reproducible new file mode 100644 index 000000000..18201d712 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/apt/apt.conf.d/99dstack-reproducible @@ -0,0 +1,3 @@ +Acquire::Check-Valid-Until "false"; +APT::Install-Recommends "false"; +APT::Install-Suggests "false"; diff --git a/os/mkosi/mkosi.skeleton/etc/docker/daemon.json b/os/mkosi/mkosi.skeleton/etc/docker/daemon.json new file mode 100644 index 000000000..83d99475a --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/docker/daemon.json @@ -0,0 +1,8 @@ +{ + "log-driver": "json-file", + "log-opts": {"max-size": "100m", "max-file": "10"}, + "runtimes": { + "nvidia": {"args": [], "path": "nvidia-container-runtime"}, + "sysbox-runc": {"path": "/usr/bin/sysbox-runc"} + } +} diff --git a/os/mkosi/mkosi.skeleton/etc/ssh/sshd_config.d/10-dstack.conf b/os/mkosi/mkosi.skeleton/etc/ssh/sshd_config.d/10-dstack.conf new file mode 100644 index 000000000..e0f62bd19 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/ssh/sshd_config.d/10-dstack.conf @@ -0,0 +1,3 @@ +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin prohibit-password diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/network/20-wired.network b/os/mkosi/mkosi.skeleton/etc/systemd/network/20-wired.network new file mode 100644 index 000000000..cc3cb0c1f --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/network/20-wired.network @@ -0,0 +1,6 @@ +[Match] +Name=en* eth* + +[Network] +DHCP=yes +IPv6AcceptRA=yes diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/debug-shell.service b/os/mkosi/mkosi.skeleton/etc/systemd/system/debug-shell.service new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/debug-shell.service @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/dstack-firstboot.service b/os/mkosi/mkosi.skeleton/etc/systemd/system/dstack-firstboot.service new file mode 100644 index 000000000..207c85abf --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/dstack-firstboot.service @@ -0,0 +1,11 @@ +[Unit] +Description=Initialize dstack immutable guest state +Before=docker.service dstack-prepare.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/mkdir -p /dstack /dstack/.host-shared /var/lib/docker /var/lib/containerd /etc/wireguard +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/getty@.service b/os/mkosi/mkosi.skeleton/etc/systemd/system/getty@.service new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/getty@.service @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/hibernate.target b/os/mkosi/mkosi.skeleton/etc/systemd/system/hibernate.target new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/hibernate.target @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/hybrid-sleep.target b/os/mkosi/mkosi.skeleton/etc/systemd/system/hybrid-sleep.target new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/hybrid-sleep.target @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/serial-getty@.service b/os/mkosi/mkosi.skeleton/etc/systemd/system/serial-getty@.service new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/serial-getty@.service @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/sleep.target b/os/mkosi/mkosi.skeleton/etc/systemd/system/sleep.target new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/sleep.target @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/etc/systemd/system/suspend.target b/os/mkosi/mkosi.skeleton/etc/systemd/system/suspend.target new file mode 120000 index 000000000..dc1dc0cde --- /dev/null +++ b/os/mkosi/mkosi.skeleton/etc/systemd/system/suspend.target @@ -0,0 +1 @@ +/dev/null \ No newline at end of file diff --git a/os/mkosi/mkosi.skeleton/usr/bin/dstack-volatile-binds.sh b/os/mkosi/mkosi.skeleton/usr/bin/dstack-volatile-binds.sh new file mode 100755 index 000000000..72ea277a9 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/bin/dstack-volatile-binds.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +for name in cache lib log spool tmp; do + source=/var/volatile/$name + target=/var/$name + mkdir -p "$source" "$target" + mountpoint -q "$target" || mount --bind "$source" "$target" +done diff --git a/os/mkosi/mkosi.skeleton/usr/lib/systemd/system-preset/80-dstack.preset b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system-preset/80-dstack.preset new file mode 100644 index 000000000..e89439659 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system-preset/80-dstack.preset @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +enable systemd-networkd.service +enable systemd-resolved.service +enable chrony.service +enable docker.service +enable var-volatile.mount +enable dstack-volatile-binds.service +enable dstack-firstboot.service +enable dstack-prepare.service +enable dstack-guest-agent.socket +enable dstack-guest-agent.service +enable app-compose.service +enable wg-checker.service +enable nvidia-persistenced.service +enable nvidia-fabricmanager.service +enable containerd-stargz-grpc.service +enable sysbox.service +enable sysbox-fs.service +enable sysbox-mgr.service +enable dstack-tee-simulator.service diff --git a/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/dstack-volatile-binds.service b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/dstack-volatile-binds.service new file mode 100644 index 000000000..84fc4477e --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/dstack-volatile-binds.service @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +[Unit] +Description=Bind volatile state over the immutable root +DefaultDependencies=no +Requires=var-volatile.mount +After=var-volatile.mount +Before=local-fs.target + +[Service] +Type=oneshot +ExecStart=/usr/bin/dstack-volatile-binds.sh +RemainAfterExit=yes + +[Install] +WantedBy=local-fs.target diff --git a/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd-persistent-storage.service.d/10-volatile-state.conf b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd-persistent-storage.service.d/10-volatile-state.conf new file mode 100644 index 000000000..810ef6722 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd-persistent-storage.service.d/10-volatile-state.conf @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +[Unit] +Requires=dstack-volatile-binds.service +After=dstack-volatile-binds.service diff --git a/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-resolved.service.d/10-var-volatile.conf b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-resolved.service.d/10-var-volatile.conf new file mode 100644 index 000000000..f27776bf6 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/systemd-resolved.service.d/10-var-volatile.conf @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +[Unit] +After=systemd-tmpfiles-setup.service var-volatile.mount +Requires=var-volatile.mount diff --git a/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/var-volatile.mount b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/var-volatile.mount new file mode 100644 index 000000000..05072403e --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/systemd/system/var-volatile.mount @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +[Unit] +Description=Volatile writable state +Before=local-fs.target + +[Mount] +What=tmpfs +Where=/var/volatile +Type=tmpfs +Options=mode=0755,nosuid,nodev + +[Install] +WantedBy=local-fs.target diff --git a/os/mkosi/mkosi.skeleton/usr/lib/tmpfiles.d/dstack-image.conf b/os/mkosi/mkosi.skeleton/usr/lib/tmpfiles.d/dstack-image.conf new file mode 100644 index 000000000..7bd3ade16 --- /dev/null +++ b/os/mkosi/mkosi.skeleton/usr/lib/tmpfiles.d/dstack-image.conf @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +d /dstack 0755 root root - +d /dstack/.host-shared 0755 root root - +d /etc/wireguard 0700 root root - +d /var/lib/docker 0755 root root - +d /var/lib/containerd 0755 root root - +d /var/volatile 0755 root root - +d /var/lib/tpm2-tss/system/keystore 0755 root root - +d /var/mail 0755 root root - +L+ /tapp - - - - dstack +L+ /usr/sbin/init - - - - ../lib/systemd/systemd +L+ /etc/resolv.conf - - - - /run/systemd/resolve/stub-resolv.conf diff --git a/os/mkosi/mkosi.skeleton/var/lib/containerd/.dstack-keep b/os/mkosi/mkosi.skeleton/var/lib/containerd/.dstack-keep new file mode 100644 index 000000000..e69de29bb diff --git a/os/mkosi/mkosi.skeleton/var/lib/docker/.dstack-keep b/os/mkosi/mkosi.skeleton/var/lib/docker/.dstack-keep new file mode 100644 index 000000000..e69de29bb diff --git a/os/mkosi/mkosi.skeleton/var/lib/tpm2-tss/system/keystore/.dstack-keep b/os/mkosi/mkosi.skeleton/var/lib/tpm2-tss/system/keystore/.dstack-keep new file mode 100644 index 000000000..e69de29bb diff --git a/os/mkosi/mkosi.skeleton/var/mail/.dstack-keep b/os/mkosi/mkosi.skeleton/var/mail/.dstack-keep new file mode 100644 index 000000000..e69de29bb diff --git a/os/mkosi/mkosi.skeleton/var/volatile/.dstack-keep b/os/mkosi/mkosi.skeleton/var/volatile/.dstack-keep new file mode 100644 index 000000000..e69de29bb diff --git a/os/mkosi/mkosi.tools.conf b/os/mkosi/mkosi.tools.conf new file mode 100644 index 000000000..c83a700e3 --- /dev/null +++ b/os/mkosi/mkosi.tools.conf @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +[Distribution] +Snapshot=20260721T000000Z +Repositories=main diff --git a/os/mkosi/parity.json b/os/mkosi/parity.json new file mode 100644 index 000000000..f8489f812 --- /dev/null +++ b/os/mkosi/parity.json @@ -0,0 +1,137 @@ +{ + "reference": "os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs.bb", + "required_rootfs_paths": { + "dstack-guest": [ + "usr/bin/dstack-guest-agent", + "usr/bin/dstack-util" + ], + "container": [ + "usr/bin/docker", + "usr/bin/containerd", + "usr/bin/nerdctl", + "usr/bin/containerd-stargz-grpc", + "etc/containerd/config.toml", + "usr/bin/docker-compose", + "usr/lib/cni/bridge", + "usr/lib/cni/host-local", + "usr/lib/systemd/system/containerd-stargz-grpc.service", + "etc/docker/daemon.json" + ], + "sysbox-0.6.7": [ + "usr/bin/sysbox-runc", + "usr/bin/sysbox-fs", + "usr/bin/sysbox-mgr", + "usr/lib/systemd/system/sysbox.service", + "usr/lib/systemd/system/sysbox-fs.service", + "usr/lib/systemd/system/sysbox-mgr.service", + "etc/sysctl.d/99-sysbox-sysctl.conf", + "etc/modules-load.d/50-sysbox-mod.conf" + ], + "zfs-2.4.0": [ + "usr/sbin/zfs", + "usr/sbin/zpool", + "usr/lib/systemd/system/zfs.target", + "usr/libexec/zfs/zed.d" + ], + "nvidia-595.58.03": [ + "usr/bin/nvidia-smi", + "usr/bin/nvidia-persistenced", + "usr/bin/nv-fabricmanager", + "usr/lib/systemd/system/nvidia-fabricmanager.service", + "usr/bin/nvidia-gpu-detect", + "usr/lib/x86_64-linux-gnu/libnvidia-nscq.so.2", + "etc/systemd/system/nvidia-fabricmanager.service.d/10-nvswitch-condition.conf" + ], + "nvidia-container": [ + "usr/bin/nvidia-container-cli", + "usr/bin/nvidia-container-runtime", + "usr/bin/nvidia-ctk", + "etc/nvidia-container-runtime/config.toml" + ], + "nvattest-2026.06.09": [ + "usr/bin/nvattest", + "usr/lib/libnvat.so.1", + "usr/share/nvattest/policies/allow_trust_outpost_ocsp.rego", + "usr/lib/systemd/system/dstack-prepare.service.d/10-nvidia-gpu-ordering.conf" + ], + "security-and-tee": [ + "usr/sbin/cryptsetup", + "usr/bin/tpm2_getrandom", + "etc/ssh/sshd_config.d/10-dstack.conf", + "usr/lib/systemd/system/var-volatile.mount", + "usr/lib/systemd/system/dstack-volatile-binds.service" + ] + }, + "required_kernel_globs": { + "nvidia": [ + "usr/lib/modules/*/kernel/drivers/video/nvidia.ko", + "usr/lib/modules/*/kernel/drivers/video/nvidia-uvm.ko", + "usr/lib/modules/*/kernel/drivers/video/nvidia-modeset.ko", + "usr/lib/modules/*/kernel/drivers/video/nvidia-drm.ko", + "usr/lib/modules/*/kernel/drivers/video/nvidia-peermem.ko" + ], + "zfs": [ + "usr/lib/modules/*/extra/zfs.ko" + ] + }, + "dev_only_paths": [ + "usr/bin/dstack-tee-simulator", + "usr/bin/strace", + "usr/bin/gdb", + "usr/sbin/sshd" + ], + "required_rootfs_globs": { + "nvidia-firmware": [ + "usr/lib/firmware/nvidia/595.58.03/*" + ], + "nvidia-libraries": [ + "usr/lib/x86_64-linux-gnu/libcuda.so.595.58.03", + "usr/lib/x86_64-linux-gnu/libnvidia-ml.so.595.58.03" + ], + "cni-plugins": [ + "usr/lib/cni/loopback", + "usr/lib/cni/portmap" + ] + }, + "required_kernel_config": [ + "CONFIG_ACPI_TABLE_UPGRADE=y", + "CONFIG_INTEL_TDX_GUEST=y", + "CONFIG_TDX_GUEST_DRIVER=y", + "CONFIG_AMD_MEM_ENCRYPT=y", + "CONFIG_SEV_GUEST=y", + "CONFIG_TCG_TPM=y", + "CONFIG_DM_CRYPT=y", + "CONFIG_DM_VERITY=y", + "CONFIG_OVERLAY_FS=y", + "CONFIG_USER_NS=y", + "CONFIG_SECCOMP_FILTER=y", + "CONFIG_BRIDGE_NETFILTER=m", + "CONFIG_TUN=m", + "CONFIG_FUSE_FS=y", + "CONFIG_NF_TABLES=m", + "CONFIG_IP_SET=m", + "CONFIG_WIREGUARD=y", + "CONFIG_DEBUG_INFO_BTF=y", + "CONFIG_CUSE=y", + "CONFIG_TCG_VTPM_PROXY=y", + "CONFIG_SECURITY_SELINUX=y", + "CONFIG_SCSI=n", + "CONFIG_INPUT=n", + "CONFIG_VT=n", + "CONFIG_WIRELESS=n", + "CONFIG_SOUND=n", + "CONFIG_USB_SUPPORT=n" + ], + "prod_forbidden_paths": [ + "usr/bin/dstack-tee-simulator", + "usr/sbin/sshd", + "usr/bin/gdb", + "usr/bin/strace" + ], + "runtime_link_paths": [ + "usr/bin/nvattest", + "usr/bin/nvidia-smi", + "usr/bin/nvidia-container-cli", + "usr/sbin/zfs" + ] +} diff --git a/os/mkosi/scripts/build-components.sh b/os/mkosi/scripts/build-components.sh new file mode 100755 index 000000000..8b6e4dee4 --- /dev/null +++ b/os/mkosi/scripts/build-components.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +# shellcheck source=/dev/null +source "$SELF/scripts/dev-cache.sh" +# shellcheck source=/dev/null +source "$SELF/scripts/component-framework.sh" + +use_cache=0 +if [[ ${1:-} == --dev-cache ]]; then use_cache=1; shift; fi +work=${1:?work directory required} +stage=${2:?rootfs staging tree required} +kstage=${3:?kernel staging tree required} +flavor=${4:?flavor required} +component_framework_init "$use_cache" "$ROOT" "$SELF" \ + "$(realpath -m "$work")" "$(realpath -m "$stage")" \ + "$(realpath -m "$kstage")" "$flavor" + +components=( + dstack-rust + image-tools + container-stack + sysbox + nvattest + kernel + nvidia + zfs + ovmf +) + +for component in "${components[@]}"; do component_run "$component"; done +component_assemble diff --git a/os/mkosi/scripts/check-kernel-config.sh b/os/mkosi/scripts/check-kernel-config.sh new file mode 100755 index 000000000..e7c42d48c --- /dev/null +++ b/os/mkosi/scripts/check-kernel-config.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +config=${1:?kernel .config required} +fragment=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/components/kernel/kernel.config +failed=0 +while IFS= read -r line; do + [[ $line =~ ^CONFIG_[A-Z0-9_]+=(y|m|n)$ ]] || continue + key=${line%%=*}; want=${line#*=} + if [[ $want == n ]]; then + grep -qx "# $key is not set" "$config" && continue + elif grep -qx "$key=$want" "$config" || { [[ $want == m ]] && grep -qx "$key=y" "$config"; }; then + continue + fi + printf 'kernel config mismatch: wanted %s\n' "$line" >&2 + failed=1 +done < "$fragment" +exit "$failed" diff --git a/os/mkosi/scripts/component-framework.sh b/os/mkosi/scripts/component-framework.sh new file mode 100644 index 000000000..e4a4b3a63 --- /dev/null +++ b/os/mkosi/scripts/component-framework.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Shared API for self-describing mkosi components. + +declare -Ag COMPONENT_KEYS=() +declare -ag COMPONENT_ROOTFS=() COMPONENT_KERNEL=() + +component_framework_init() { + COMPONENT_CACHE=$1 + ROOT=$2 SELF=$3 WORK=$4 STAGE=$5 KSTAGE=$6 FLAVOR=$7 + COMPONENT_DIR="$SELF/components" + dev_cache_init "$COMPONENT_CACHE" +} + +key_value() { printf 'value\0%s\0' "$*"; } + +key_file() { + local file + for file; do + printf 'file\0%s\0' "${file#"$ROOT/"}" + sha256sum "$file" + done +} + +key_tree() { + local kind input value path matches + [[ -f ${DSTACK_SOURCE_MANIFEST:?} ]] || { + echo "development source manifest is missing" >&2 + return 1 + } + while IFS= read -r -d '' kind && + IFS= read -r -d '' input && + IFS= read -r -d '' value; do + matches=0 + for path; do + path=${path%/} + if [[ $input == "$path" || $input == "$path/"* ]]; then + matches=1 + break + fi + done + [[ $matches == 1 ]] || continue + case "$kind" in + file) key_file "$ROOT/$input" ;; + gitlink) printf 'gitlink\0%s\0%s\0' "$input" "$value" ;; + missing) printf 'missing\0%s\0' "$input" ;; + *) echo "invalid source manifest entry: $kind" >&2; return 1 ;; + esac + done < "$DSTACK_SOURCE_MANIFEST" +} + +key_tools() { + local tool path package + for tool; do + path=$(readlink -f "$(command -v "$tool")") + printf 'tool\0%s\0%s\0' "$tool" "$path" + sha256sum "$path" + case "$tool" in + gcc|ld|cmake|make|autoconf|automake|tar) "$tool" --version | head -1 ;; + go) go version ;; + *) "$tool" --version ;; + esac + package=$(dpkg-query -S "$path" 2>/dev/null | head -1 | cut -d: -f1 || true) + [[ -z $package ]] || key_packages "$package" + done +} + +key_packages() { + local package + for package; do + dpkg-query -W -f='package=${binary:Package}=${Version}\n' "$package" + done | sort +} + +key_dependency() { + local dependency=$1 + [[ -n ${COMPONENT_KEYS[$dependency]:-} ]] || { + echo "component dependency not built: $dependency" >&2; return 1; + } + printf 'dependency\0%s\0%s\0' "$dependency" "${COMPONENT_KEYS[$dependency]}" +} + +component_run() { + local name=$1 definition key + COMPONENT_PATH="$COMPONENT_DIR/$name" + definition="$COMPONENT_PATH/$name.sh" + COMPONENT_NAME='' + COMPONENT_CACHE_PATHS=() + COMPONENT_ROOTFS_TREES=() + COMPONENT_KERNEL_TREES=() + unset -f component_cache_key component_build component_prepare_outputs 2>/dev/null || true + # shellcheck source=/dev/null + source "$definition" + [[ $COMPONENT_NAME == "$name" ]] + declare -F component_cache_key >/dev/null + declare -F component_build >/dev/null + + # Production does not read or write the component cache, so it must not + # inspect cache-only inputs either. Besides avoiding wasted work, this + # keeps production builds independent of linked-worktree Git metadata that + # is intentionally not mounted into mkosi's source sandbox. + if [[ $COMPONENT_CACHE == 1 ]]; then + key=$({ + key_value component-schema-v1 "$COMPONENT_NAME" "$FLAVOR" \ + "${SOURCE_DATE_EPOCH:?}" "$(uname -m)" + key_file "$definition" "$SELF/scripts/component-framework.sh" \ + "$SELF/scripts/dev-cache.sh" "$SELF/scripts/install-toolchains.sh" \ + "$SELF/mkosi.build" "$SELF/mkosi.conf" "$SELF/mkosi.tools.conf" + component_cache_key + } | sha256sum | cut -d' ' -f1) + else + key=disabled + fi + COMPONENT_KEYS[$name]=$key + # shellcheck disable=SC2317 # Invoked indirectly by dev_cache_run. + component_build_cached() { + component_build + if declare -F component_prepare_outputs >/dev/null; then + component_prepare_outputs + fi + } + dev_cache_run "$name" "$key" "$WORK" "${COMPONENT_CACHE_PATHS[@]}" -- \ + component_build_cached + + local tree + for tree in "${COMPONENT_ROOTFS_TREES[@]}"; do + COMPONENT_ROOTFS+=("$name=$WORK/$tree") + done + for tree in "${COMPONENT_KERNEL_TREES[@]}"; do + COMPONENT_KERNEL+=("$name=$WORK/$tree") + done +} + +component_assemble() { + "$SELF/scripts/merge-component-trees.py" "$STAGE" "${COMPONENT_ROOTFS[@]}" + "$SELF/scripts/merge-component-trees.py" "$KSTAGE" "${COMPONENT_KERNEL[@]}" +} diff --git a/os/mkosi/scripts/dev-cache.sh b/os/mkosi/scripts/dev-cache.sh new file mode 100644 index 000000000..92d757796 --- /dev/null +++ b/os/mkosi/scripts/dev-cache.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# Local archive mechanics for dev-image. Component definitions own their keys +# and output boundaries; this file only validates, restores and stores them. + +DEV_CACHE_ENABLED=0 + +dev_cache_init() { + DEV_CACHE_ENABLED=$1 +} + +dev_cache_run() { + local component=$1 key=$2 base=$3 + shift 3 + local outputs=() + while [[ $1 != -- ]]; do outputs+=("$1"); shift; done + shift + + if [[ $DEV_CACHE_ENABLED != 1 ]]; then + "$@" + return + fi + + local cache_root=${DSTACK_DEV_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/dstack/mkosi-dev} + local dir="$cache_root/$component" archive checksum + archive="$dir/$key.tar.zst" + checksum="$archive.sha256" + mkdir -p "$dir" "$base" + + ( + flock 9 + if [[ -f $archive && -f $checksum ]] && + (cd "$dir" && sha256sum --check --status "${checksum##*/}"); then + tar --zstd -xf "$archive" -C "$base" + echo "development cache hit: $component" + exit + fi + + local tmp + rm -f "$archive" "$checksum" + echo "development cache miss: $component" + "$@" + tmp=$(mktemp "$dir/.${key}.XXXXXX.tar.zst") + tar --zstd -cf "$tmp" -C "$base" "${outputs[@]}" + mv "$tmp" "$archive" + (cd "$dir" && sha256sum "${archive##*/}" > "${checksum##*/}.tmp") + mv "$checksum.tmp" "$checksum" + ) 9>"$dir/$key.lock" +} diff --git a/os/mkosi/scripts/install-toolchains.sh b/os/mkosi/scripts/install-toolchains.sh new file mode 100755 index 000000000..3e09af348 --- /dev/null +++ b/os/mkosi/scripts/install-toolchains.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +# shellcheck source=/dev/null +source "$ROOT/os/mkosi/versions.env" +DOWNLOADS=$(realpath -m "${1:?download directory required}") +PREFIX=/opt/dstack-toolchains +mkdir -p "$DOWNLOADS" "$PREFIX" + +fetch() { + local url=$1 sha=$2 output=$3 + if [[ ! -f $output ]]; then + curl --fail --location --retry 3 --retry-all-errors \ + --connect-timeout 30 --max-time 600 -o "$output.tmp" "$url" + mv "$output.tmp" "$output" + fi + echo "$sha $output" | sha256sum --check --status || { + echo "toolchain checksum mismatch: $output" >&2 + exit 1 + } +} + +install_rust_component() { + local component=$1 sha=$2 archive base dir + archive="$DOWNLOADS/$component-$RUST_TOOLCHAIN_VERSION-x86_64-unknown-linux-gnu.tar.xz" + fetch "https://static.rust-lang.org/dist/${archive##*/}" "$sha" "$archive" + base=${archive##*/} + dir=$(mktemp -d "/var/tmp/$component.XXXXXX") + tar -C "$dir" -xf "$archive" + "$dir/${base%.tar.xz}/install.sh" \ + --prefix="$PREFIX/rust" --disable-ldconfig >/dev/null + rm -rf "$dir" +} + +install_rust_component rustc "$RUSTC_TOOLCHAIN_SHA256" +install_rust_component cargo "$CARGO_TOOLCHAIN_SHA256" +install_rust_component rust-std "$RUST_STD_TOOLCHAIN_SHA256" + +go_archive="$DOWNLOADS/go$GO_TOOLCHAIN_VERSION.linux-amd64.tar.gz" +fetch "https://go.dev/dl/${go_archive##*/}" "$GO_TOOLCHAIN_SHA256" "$go_archive" +rm -rf "$PREFIX/go" +tar -C "$PREFIX" -xf "$go_archive" + +export PATH="$PREFIX/rust/bin:$PREFIX/go/bin:$PATH" +rustc --version +cargo --version +go version diff --git a/os/mkosi/scripts/make-release-artifacts.sh b/os/mkosi/scripts/make-release-artifacts.sh new file mode 100755 index 000000000..adda3633d --- /dev/null +++ b/os/mkosi/scripts/make-release-artifacts.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +umask 0022 +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +SELF="$ROOT/os/mkosi" +# shellcheck source=/dev/null +source "$SELF/versions.env" +TREE=${1:?mkosi root tree required} +KERNEL_TREE=${2:?kernel staging tree required} +OUT=${3:?artifact output directory required} +FLAVOR=${4:?flavor required} +SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:?} +export SOURCE_DATE_EPOCH TZ=UTC LC_ALL=C +mkdir -p "$OUT/files" + +kernel="$KERNEL_TREE/usr/lib/modules/$KERNEL_VERSION-dstack/vmlinuz" +install -m0644 "$kernel" "$OUT/files/bzImage" + +# Combined squashfs + dm-verity tree, matching Yocto's separate-hash=0 output. +rootfs="$OUT/files/rootfs.squashfs.verity" +# mksquashfs -noappend overwrites its filesystem but does not reliably remove +# a longer dm-verity tail left by a previous invocation. +truncate -s 0 "$rootfs" +# Privileged mkosi runs can inherit host default ACLs from their workspace. +# The guest rootfs does not rely on xattrs, so exclude this host-only metadata. +# A sorted tar stream also removes backing-filesystem directory/inode order and +# hardlink topology from the input. A single compressor worker then avoids +# host-CPU-dependent fragment ordering. +(cd "$TREE" && tar --sort=name --format=gnu \ + --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --numeric-owner \ + --mode=g-s --hard-dereference -cf - .) | \ + env -u SOURCE_DATE_EPOCH mksquashfs - "$rootfs" -tar \ + -noappend -all-root -no-progress -exports -no-hardlinks -no-tailends \ + -no-xattrs -processors 1 -comp zstd -mkfs-time "$SOURCE_DATE_EPOCH" \ + -all-time "$SOURCE_DATE_EPOCH" >/dev/null +data_size=$(stat -c %s "$rootfs") +data_size=$(( (data_size + 4095) / 4096 * 4096 )) +truncate -s "$data_size" "$rootfs" +verity_output=$(veritysetup format "$rootfs" "$rootfs" \ + --hash-offset="$data_size" --data-block-size=4096 --hash-block-size=4096 \ + --uuid=00000000-0000-0000-0000-000000000000 \ + --salt="$(printf '%064x' 0)") +root_hash=$(awk '/Root hash:/ {print $3}' <<<"$verity_output") +[[ $root_hash =~ ^[0-9a-f]{64}$ ]] || { echo "failed to obtain verity root hash" >&2; exit 1; } + +# Minimal custom initramfs that implements the same dstack.rootfs_* protocol. +ird=$(mktemp -d "$OUT/.initramfs.XXXXXX") +trap 'rm -rf "$ird"' EXIT +mkdir -p "$ird"/{bin,sbin,dev,proc,sys,run,root,usr/bin,usr/sbin,lib,lib64} +install -m0755 "$TREE/usr/bin/busybox" "$ird/bin/busybox" +for applet in sh mount mkdir cat grep head cut basename realpath switch_root; do + ln -s busybox "$ird/bin/$applet" +done +verity_bin=$(find "$TREE" -type f \( -path '*/sbin/veritysetup' -o -path '*/bin/veritysetup' \) | head -1) +[[ -n $verity_bin ]] || { echo 'veritysetup missing from mkosi tree' >&2; exit 1; } +install -m0755 "$verity_bin" "$ird/sbin/veritysetup" +command -v lddtree >/dev/null || { echo 'lddtree (pax-utils) is required' >&2; exit 1; } +while read -r lib; do + [[ $lib == "$verity_bin" ]] && continue + rel=${lib#"$TREE"} + [[ $rel == /* ]] || continue + install -Dm0755 "$lib" "$ird$rel" +done < <(lddtree -l -R "$TREE" "${verity_bin#"$TREE"}") +install -m0755 "$ROOT/os/yocto/layers/meta-dstack/recipes-core/images/dstack-initscript/init" "$ird/init" +find "$ird" -print0 | xargs -0 touch -h -d "@$SOURCE_DATE_EPOCH" +(cd "$ird" && find . -print0 | sort -z | \ + cpio --null -o --format=newc --reproducible --owner=0:0 2>/dev/null) \ + | gzip -n -9 > "$OUT/files/initramfs.cpio.gz" + +# OVMF is the pinned TDX build; the EFI stub comes from the Debian snapshot. +ovmf="$KERNEL_TREE/ovmf.fd" +ovmf_sev="$KERNEL_TREE/ovmf-sev.fd" +stub=$(find "$TREE/usr/lib/systemd/boot/efi" -type f -name 'linuxx64.efi.stub' | head -1) +[[ -f $ovmf && -f $ovmf_sev && -f $stub ]] || { + echo 'TDX/SEV OVMF or systemd EFI stub missing' >&2; exit 1; +} +install -m0644 "$ovmf" "$OUT/files/ovmf.fd" +install -m0644 "$ovmf_sev" "$OUT/files/ovmf-sev.fd" + +cmdline="console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0 mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n tsc=reliable no-kvmclock dstack.rootfs_hash=$root_hash dstack.rootfs_size=$data_size" +ukify build --efi-arch=x64 --stub="$stub" --linux="$kernel" \ + --initrd="$OUT/files/initramfs.cpio.gz" --cmdline="$cmdline" \ + --os-release="@$TREE/usr/lib/os-release" \ + --output="$OUT/files/dstack-uki.efi" +touch -d "@$SOURCE_DATE_EPOCH" "$OUT/files/"* + +is_dev=false; name=dstack +if [[ $FLAVOR == dev ]]; then is_dev=true; name=dstack-dev; fi +python3 - "$OUT/artifact-manifest.json" "$name" "$FLAVOR" "$is_dev" \ + "${DSTACK_SOURCE_REVISION:?}" "$root_hash" "$data_size" "$DSTACK_VERSION" "$OVMF_VARIANT" <<'PY' +import json, sys +path,name,flavor,is_dev,revision,root_hash,data_size,version,ovmf_variant=sys.argv[1:] +data={"schema_version":1,"backend":"mkosi","image":{"name":name,"version":version, +"flavor":flavor,"is_dev":is_dev=="true"},"source":{"git_revision":revision}, +"boot":{"ovmf_variant":ovmf_variant},"verity":{"root_hash":root_hash,"data_size":data_size}, +"artifacts":{"initramfs":"files/initramfs.cpio.gz","kernel":"files/bzImage", +"firmware":"files/ovmf.fd","rootfs_verity":"files/rootfs.squashfs.verity", +"firmware_sev":"files/ovmf-sev.fd","uki":"files/dstack-uki.efi"}, +"backend_metadata":{"machine":"dstack","distribution":"debian"}} +with open(path,"w") as f: json.dump(data,f,indent=2); f.write("\n") +PY diff --git a/os/mkosi/scripts/merge-component-trees.py b/os/mkosi/scripts/merge-component-trees.py new file mode 100755 index 000000000..e4a19e4a1 --- /dev/null +++ b/os/mkosi/scripts/merge-component-trees.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Merge component install trees while rejecting every non-directory clash.""" + +import os +import shutil +import stat +import sys +from pathlib import Path + + +def exists(path: Path) -> bool: + """Return whether a path exists, including a dangling symlink.""" + return path.exists() or path.is_symlink() + + +def merge(component: str, source: Path, destination: Path) -> None: + """Merge one component tree into the destination without overwrites.""" + if not source.is_dir(): + raise SystemExit(f"{component}: missing install tree: {source}") + for root, directories, files in os.walk(source): + directories.sort() + files.sort() + relative = Path(root).relative_to(source) + target_root = destination / relative + target_root.mkdir(parents=True, exist_ok=True) + for name in directories: + src = Path(root) / name + dst = target_root / name + if src.is_symlink(): + files.append(name) + continue + if exists(dst) and not dst.is_dir(): + raise SystemExit(f"component install conflict: {component}: {dst}") + dst.mkdir(exist_ok=True) + for name in files: + src = Path(root) / name + dst = target_root / name + if exists(dst): + raise SystemExit(f"component install conflict: {component}: {dst}") + mode = src.lstat().st_mode + if stat.S_ISLNK(mode): + dst.symlink_to(os.readlink(src)) + elif stat.S_ISREG(mode): + shutil.copy2(src, dst, follow_symlinks=False) + else: + raise SystemExit(f"{component}: unsupported install entry: {src}") + + +def main() -> None: + """Merge the component specifications passed on the command line.""" + destination = Path(sys.argv[1]) + destination.mkdir(parents=True, exist_ok=True) + for specification in sys.argv[2:]: + component, source = specification.split("=", 1) + merge(component, Path(source), destination) + + +if __name__ == "__main__": + main() diff --git a/os/mkosi/scripts/normalize-skeleton-modes.sh b/os/mkosi/scripts/normalize-skeleton-modes.sh new file mode 100755 index 000000000..c872225ea --- /dev/null +++ b/os/mkosi/scripts/normalize-skeleton-modes.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +SELF=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +SKELETON="$SELF/mkosi.skeleton" +TREE=${1:?rootfs tree required} + +# Git tracks regular files as either non-executable or executable, but a +# shared checkout can add group-write bits. mkosi correctly preserves source +# modes, so canonicalize only those two Git-representable classes after the +# native SkeletonTrees= copy. +while IFS= read -r -d '' source; do + relative=${source#"$SKELETON/"} + destination="$TREE/$relative" + [[ -e $destination ]] || { + echo "skeleton path missing from rootfs: $relative" >&2 + exit 1 + } + if [[ -x $source ]]; then + chmod 0755 "$destination" + else + chmod 0644 "$destination" + fi +done < <(find "$SKELETON" -type f ! -name .dstack-keep -print0) + +while IFS= read -r -d '' directory; do + relative=${directory#"$SKELETON"} + chmod 0755 "$TREE$relative" +done < <(find "$SKELETON" -type d -print0) diff --git a/os/mkosi/scripts/write-source-manifest.py b/os/mkosi/scripts/write-source-manifest.py new file mode 100755 index 000000000..29823b9cd --- /dev/null +++ b/os/mkosi/scripts/write-source-manifest.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: Apache-2.0 +"""Describe Git-owned and untracked source paths for development cache keys.""" + +import os +import pathlib +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]).resolve() +output = pathlib.Path(sys.argv[2]).resolve() +entries = subprocess.check_output( + ["git", "-C", root, "ls-files", "-z", "--cached", "--others", "--exclude-standard"] +).split(b"\0") +stage = subprocess.check_output( + ["git", "-C", root, "ls-files", "-z", "--stage", "--cached"] +).split(b"\0") +gitlinks = { + record.split(b"\t", 1)[1]: record.split(b" ", 1)[1].split(b" ", 1)[0] + for record in stage + if record.startswith(b"160000 ") +} + +output.parent.mkdir(parents=True, exist_ok=True) +temporary = output.with_name(f".{output.name}.{os.getpid()}.tmp") +with temporary.open("wb") as manifest: + for encoded in sorted(filter(None, entries)): + path = root / os.fsdecode(encoded) + if encoded in gitlinks and path.is_dir(): + revision = subprocess.check_output( + ["git", "-C", path, "rev-parse", "HEAD"] + ).strip() + kind, value = b"gitlink", revision + elif path.is_file(): + kind, value = b"file", b"" + else: + kind, value = b"missing", b"" + manifest.write(kind + b"\0" + encoded + b"\0" + value + b"\0") +temporary.replace(output) diff --git a/os/mkosi/tests/acceptance.sh b/os/mkosi/tests/acceptance.sh new file mode 100755 index 000000000..1753a6084 --- /dev/null +++ b/os/mkosi/tests/acceptance.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +D=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +for component in dstack-rust image-tools container-stack sysbox nvattest kernel nvidia zfs ovmf; do + test -f "$D/components/$component/$component.sh" + test -x "$D/components/$component/$component-build.sh" +done +# shellcheck source=/dev/null +source "$D/versions.env" +[[ $KERNEL_VERSION == 7.* && $KERNEL_VERSION != *-rc* ]] +[[ $KERNEL_SHA256 =~ ^[0-9a-f]{64}$ ]] +grep -q "^Snapshot=$DEBIAN_SNAPSHOT" "$D/mkosi.conf" +grep -q '^Format=tar$' "$D/mkosi.conf" +grep -q '^Output=dstack-0.6.0$' "$D/mkosi.conf" +grep -q '^CompressOutput=gz$' "$D/mkosi.conf" +grep -q '^SplitArtifacts=$' "$D/mkosi.conf" +grep -q '^Bootable=no$' "$D/mkosi.conf" +grep -q '^Bootloader=none$' "$D/mkosi.conf" +for key in ACPI ACPI_TABLE_UPGRADE INTEL_TDX_GUEST TDX_GUEST_DRIVER \ + AMD_MEM_ENCRYPT SEV_GUEST TCG_TPM VIRTIO_PCI VIRTIO_NET BLK_DEV_NVME \ + DM_CRYPT DM_VERITY OVERLAY_FS CGROUPS USER_NS SECCOMP BPF_SYSCALL \ + NF_TABLES VSOCKETS HARDENED_USERCOPY; do + grep -Eq "^CONFIG_${key}=(y|m)$" "$D/components/kernel/kernel.config" || { echo "missing CONFIG_$key"; exit 1; } +done +grep -q '0002-acpi-sandbox' "$D/components/kernel/kernel-build.sh" +grep -q -- '--fuzz=0' "$D/components/kernel/kernel-build.sh" +for service in dstack-guest-agent dstack-prepare app-compose wg-checker; do + grep -q "$service" "$D/mkosi.skeleton/usr/lib/systemd/system-preset/80-dstack.preset" +done +grep -q 'artifact-manifest.json' "$D/scripts/make-release-artifacts.sh" +grep -q -- '--hard-dereference' "$D/scripts/make-release-artifacts.sh" +grep -q -- '--mode=g-s' "$D/scripts/make-release-artifacts.sh" +grep -q 'mksquashfs -.*-tar' "$D/scripts/make-release-artifacts.sh" +grep -q 'image/assemble.sh' "$D/mkosi.postoutput" +grep -q 'ovmf-build.sh' "$D/components/ovmf/ovmf.sh" +grep -q 'fbe0805b2091393406952e84724188f8c1941837' "$D/components/ovmf/ovmf-build.sh" +grep -q 'AmdSev/AmdSevX64.dsc' "$D/components/ovmf/ovmf-build.sh" +grep -q '0006-OvmfPkg-AmdSev-drop-embedded-grub.patch' "$D/components/ovmf/ovmf.sh" +grep -q 'objcopy --strip-debug' "$D/mkosi.build" +grep -q 'depmod -b.*KERNEL_VERSION-dstack' "$D/mkosi.build" +grep -q '^CleanPackageMetadata=yes$' "$D/mkosi.conf" +grep -q '^WithDocs=no$' "$D/mkosi.conf" +grep -q '/var/lib/docker/.dstack-keep' "$D/mkosi.conf" +grep -q '! -name .dstack-keep' "$D/scripts/normalize-skeleton-modes.sh" +grep -q '/var/lib/tpm2-tss/system/keystore 0755' \ + "$D/mkosi.skeleton/usr/lib/tmpfiles.d/dstack-image.conf" +grep -q '/var/lib/dpkg' "$D/mkosi.profiles/prod/mkosi.conf" +if grep -q '^[[:space:]]*ovmf$' "$D/mkosi.conf"; then + echo 'distribution OVMF must not be installed in the guest rootfs' >&2 + exit 1 +fi +grep -q '"firmware_sev":"files/ovmf-sev.fd"' "$D/scripts/make-release-artifacts.sh" +grep -q 'measurement.snp.cbor' "$D/tests/check-output.sh" +grep -q '0001-validate-ocsp-response-freshness.patch' "$D/components/nvattest/nvattest-build.sh" +grep -q -- '--fuzz=0' "$D/components/nvattest/nvattest-build.sh" +grep -q '0001-pin-fetchcontent-inputs.patch' "$D/components/nvattest/nvattest-build.sh" +grep -q 'COMPONENT_PATH/patches/0001-pin-fetchcontent-inputs.patch' "$D/components/nvattest/nvattest.sh" +grep -q '^NVATTEST_REVISION=9d12801cea8a198ea0f29640dfaf8a4017c841c5$' "$D/versions.env" +grep -q '0001-linux-7.1-drop-legacy-of-gpio-api.patch' "$D/components/nvidia/nvidia-build.sh" +grep -q '^NVIDIA_VERSION=595.58.03$' "$D/versions.env" +grep -q '^STARGZ_VERSION=0.18.2$' "$D/versions.env" +grep -q '^NERDCTL_VERSION=2.2.1$' "$D/versions.env" +grep -q '^[[:space:]]*docker-cli$' "$D/mkosi.conf" +grep -q '^SYSBOX_VERSION=0.6.7$' "$D/versions.env" +grep -q '^ZFS_VERSION=2.4.0$' "$D/versions.env" +grep -q '0001-linux-6.19-7.1-compat.patch' "$D/components/zfs/zfs-build.sh" +grep -q -- '--fuzz=0' "$D/components/zfs/zfs-build.sh" +python3 -m json.tool "$D/parity.json" >/dev/null +grep -q 'rootfs.img.parted.verity' "$D/../image/assemble.sh" +bash -n "$D"/*.sh "$D"/mkosi.build "$D"/mkosi.clean "$D"/mkosi.finalize \ + "$D"/mkosi.postoutput "$D"/scripts/*.sh "$D"/components/*/*.sh "$D"/tests/*.sh +grep -q 'Archiving UKI image' "$D/../image/assemble.sh" +grep -Fq "FLAVORS=\${FLAVORS:-prod}" "$D/build.sh" +grep -q 'DSTACK_COMPONENT_CACHE.*dev-image' "$D/build.sh" +grep -q 'write-source-manifest.py' "$D/build.sh" +grep -q 'DSTACK_SOURCE_REVISION.*revision' "$D/build.sh" +grep -q 'DSTACK_SOURCE_REVISION:?' "$D/scripts/make-release-artifacts.sh" +grep -q 'scripts/build-components.sh' "$D/mkosi.build" +grep -q '^BuildPackages=' "$D/mkosi.conf" +grep -q '^ToolsTree=yes$' "$D/mkosi.conf" +grep -q '^ToolsTreeProfiles=misc$' "$D/mkosi.conf" +grep -q '^Snapshot=20260721T000000Z$' "$D/mkosi.tools.conf" +grep -q 'clean -f' "$D/build.sh" +grep -q 'install-toolchains.sh' "$D/mkosi.build" +grep -q '^RUST_TOOLCHAIN_VERSION=1.92.0$' "$D/versions.env" +grep -q '^GO_TOOLCHAIN_VERSION=1.22.2$' "$D/versions.env" +for component in dstack-rust image-tools container-stack sysbox nvattest kernel nvidia zfs ovmf; do + definition="$D/components/$component/$component.sh" + grep -q "^COMPONENT_NAME=$component$" "$definition" + grep -q '^component_cache_key()' "$definition" + grep -q '^component_build()' "$definition" + grep -q '^COMPONENT_CACHE_PATHS=' "$definition" +done +"$D/tests/test-dev-cache.sh" +"$D/tests/test-component-framework.sh" +"$D/tests/test-component-merge.sh" +echo 'mkosi static acceptance passed' diff --git a/os/mkosi/tests/check-output.sh b/os/mkosi/tests/check-output.sh new file mode 100755 index 000000000..7f3b3aae8 --- /dev/null +++ b/os/mkosi/tests/check-output.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +out=${1:?assembled output directory required} +flavor=${2:-prod} +for file in bzImage initramfs.cpio.gz ovmf.fd ovmf-sev.fd rootfs.img.parted.verity \ + metadata.json measurement.tdx.cbor measurement.snp.cbor sha256sum.txt digest.txt; do + [[ -s $out/$file ]] || { echo "missing compatible output: $file" >&2; exit 1; } +done +(cd "$out" && sha256sum -c sha256sum.txt) +grep -q 'dstack-rootfs' < <(sgdisk -p "$out/rootfs.img.parted.verity") +python3 - "$out/metadata.json" "$flavor" <<'PY' +import json,sys +d=json.load(open(sys.argv[1])) +for k in ("bios","bios-sev","kernel","cmdline","initrd","rootfs","version","git_revision","is_dev","ovmf_variant"): + assert k in d, k +assert d["bios-sev"] == "ovmf-sev.fd" +assert d["is_dev"] == (sys.argv[2] == "dev") +PY +echo "Yocto-compatible release format accepted: $out" diff --git a/os/mkosi/tests/check-parity.py b/os/mkosi/tests/check-parity.py new file mode 100755 index 000000000..ece057d03 --- /dev/null +++ b/os/mkosi/tests/check-parity.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Verify Yocto-visible functional parity in an assembled mkosi tree.""" + +import glob +import json +import os +import subprocess +import sys + +spec_path, rootfs, kernel_tree, flavor = sys.argv[1:] +with open(spec_path, encoding="utf-8") as f: + spec = json.load(f) +missing = [] +for component, paths in spec["required_rootfs_paths"].items(): + for path in paths: + if not os.path.lexists(os.path.join(rootfs, path)): + missing.append(f"{component}:/{path}") +for component, patterns in spec.get("required_rootfs_globs", {}).items(): + for pattern in patterns: + if not glob.glob(os.path.join(rootfs, pattern)): + missing.append(f"{component}:/{pattern}") +for component, patterns in spec["required_kernel_globs"].items(): + for pattern in patterns: + if not glob.glob(os.path.join(kernel_tree, pattern)): + missing.append(f"{component}: kernel module ({pattern})") +configs = glob.glob(os.path.join(kernel_tree, "usr/lib/modules/*/config")) +if len(configs) != 1: + missing.append("kernel-config: expected exactly one installed config") +else: + with open(configs[0], encoding="utf-8") as f: + config = set(f.read().splitlines()) + for setting in spec.get("required_kernel_config", []): + symbol, value = setting.split("=", 1) + effective = f"# {symbol} is not set" if value == "n" else setting + if effective not in config: + missing.append(f"kernel-config: {setting}") +if flavor == "dev": + for path in spec["dev_only_paths"]: + if not os.path.lexists(os.path.join(rootfs, path)): + missing.append(f"dev:/{path}") +else: + for path in spec.get("prod_forbidden_paths", []): + if os.path.lexists(os.path.join(rootfs, path)): + missing.append(f"prod contains dev-only path:/{path}") +for path in spec.get("runtime_link_paths", []): + result = subprocess.run( + ["lddtree", "-R", rootfs, f"/{path}"], + capture_output=True, + check=False, + text=True, + ) + output = result.stdout + result.stderr + if result.returncode or "not found" in output or "did not match" in output: + missing.append(f"runtime linkage:/{path}") +if missing: + raise SystemExit("Yocto parity check failed:\n " + "\n ".join(missing)) +print("Yocto functional parity paths accepted") diff --git a/os/mkosi/tests/test-component-framework.sh b/os/mkosi/tests/test-component-framework.sh new file mode 100755 index 000000000..f98564e4c --- /dev/null +++ b/os/mkosi/tests/test-component-framework.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +D=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/project/components/base" "$tmp/project/components/dependent" +printf one > "$tmp/project/input" +printf 'file\0project/input\0\0' > "$tmp/source-manifest" + +cat > "$tmp/project/components/base/base.sh" <<'EOF_COMPONENT' +COMPONENT_NAME=base +COMPONENT_CACHE_PATHS=(stages/base) +COMPONENT_ROOTFS_TREES=(stages/base) +COMPONENT_KERNEL_TREES=() +component_cache_key() { key_tree project; } +component_build() { + mkdir -p "$WORK/stages/base" + printf x >> "$ROOT/base-count" + cp "$ROOT/project/input" "$WORK/stages/base/value" +} +EOF_COMPONENT +cat > "$tmp/project/components/dependent/dependent.sh" <<'EOF_COMPONENT' +COMPONENT_NAME=dependent +COMPONENT_CACHE_PATHS=(stages/dependent) +COMPONENT_ROOTFS_TREES=(stages/dependent) +COMPONENT_KERNEL_TREES=() +component_cache_key() { key_dependency base; } +component_build() { + mkdir -p "$WORK/stages/dependent" + printf x >> "$ROOT/dependent-count" + cp "$WORK/stages/base/value" "$WORK/stages/dependent/value" +} +EOF_COMPONENT +cat > "$tmp/run.sh" <<'EOF_DRIVER' +#!/bin/bash +set -euo pipefail +D=$1 ROOT=$2 WORK=$3 +SELF=$D +source "$D/scripts/dev-cache.sh" +source "$D/scripts/component-framework.sh" +export SOURCE_DATE_EPOCH=1 DSTACK_DEV_CACHE_DIR="$ROOT/cache" +export DSTACK_SOURCE_MANIFEST="$ROOT/source-manifest" +component_framework_init 1 "$ROOT" "$SELF" "$WORK" "$ROOT/rootfs" \ + "$ROOT/kernel" prod +COMPONENT_DIR="$ROOT/project/components" +component_run base +component_run dependent +EOF_DRIVER +chmod +x "$tmp/run.sh" + +"$tmp/run.sh" "$D" "$tmp" "$tmp/work-1" +[[ $(cat "$tmp/base-count") == x ]] +[[ $(cat "$tmp/dependent-count") == x ]] +"$tmp/run.sh" "$D" "$tmp" "$tmp/work-2" +[[ $(cat "$tmp/base-count") == x ]] +[[ $(cat "$tmp/dependent-count") == x ]] +printf two > "$tmp/project/input" +"$tmp/run.sh" "$D" "$tmp" "$tmp/work-3" +[[ $(cat "$tmp/base-count") == xx ]] +[[ $(cat "$tmp/dependent-count") == xx ]] +[[ $(cat "$tmp/work-3/stages/dependent/value") == two ]] +echo 'component framework tests passed' + +# Cache key declarations are development-only policy. A production build must +# neither evaluate them nor require access to their source metadata. +sed -i 's/component_framework_init 1 /component_framework_init 0 /' "$tmp/run.sh" +cat >> "$tmp/project/components/base/base.sh" <<'EOF_COMPONENT' +component_cache_key() { echo 'production evaluated cache key' >&2; return 1; } +EOF_COMPONENT +rm -rf "$tmp/work-4" +"$tmp/run.sh" "$D" "$tmp" "$tmp/work-4" +[[ $(cat "$tmp/work-4/stages/base/value") == two ]] +echo 'production component framework tests passed' diff --git a/os/mkosi/tests/test-component-merge.sh b/os/mkosi/tests/test-component-merge.sh new file mode 100755 index 000000000..1b51a7561 --- /dev/null +++ b/os/mkosi/tests/test-component-merge.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +D=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +mkdir -p "$tmp/a/usr/bin" "$tmp/b/usr/lib" "$tmp/c/usr/bin" +printf a > "$tmp/a/usr/bin/a" +printf b > "$tmp/b/usr/lib/b" +printf conflict > "$tmp/c/usr/bin/a" +"$D/scripts/merge-component-trees.py" "$tmp/root" \ + "a=$tmp/a" "b=$tmp/b" +[[ $(cat "$tmp/root/usr/bin/a") == a ]] +[[ $(cat "$tmp/root/usr/lib/b") == b ]] + +if "$D/scripts/merge-component-trees.py" "$tmp/root" "c=$tmp/c" \ + >"$tmp/conflict.log" 2>&1; then + echo 'component conflict was not rejected' >&2 + exit 1 +fi +grep -q 'component install conflict: c:' "$tmp/conflict.log" +echo 'component merge tests passed' diff --git a/os/mkosi/tests/test-dev-cache.sh b/os/mkosi/tests/test-dev-cache.sh new file mode 100755 index 000000000..deabb7434 --- /dev/null +++ b/os/mkosi/tests/test-dev-cache.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +D=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# shellcheck source=/dev/null +source "$D/scripts/dev-cache.sh" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +export DSTACK_DEV_CACHE_DIR="$tmp/cache" +base="$tmp/work" +count="$tmp/count" + +build_output() { + mkdir -p "$base/stage" + printf 'payload:%s\n' "$1" > "$base/stage/file" + printf x >> "$count" +} + +dev_cache_init 1 +dev_cache_run component key-v1 "$base" stage -- build_output v1 +[[ $(cat "$base/stage/file") == payload:v1 && $(cat "$count") == x ]] + +rm -rf "$base" +dev_cache_run component key-v1 "$base" stage -- build_output wrong +[[ $(cat "$base/stage/file") == payload:v1 && $(cat "$count") == x ]] + +rm -rf "$base" +dev_cache_run component key-v2 "$base" stage -- build_output v2 +[[ $(cat "$base/stage/file") == payload:v2 && $(cat "$count") == xx ]] + +rm -rf "$base" +archive=$(find "$DSTACK_DEV_CACHE_DIR/component" -name '*.tar.zst' -newer "$count") +printf corrupt >> "$archive" +dev_cache_run component key-v2 "$base" stage -- build_output rebuilt +[[ $(cat "$base/stage/file") == payload:rebuilt && $(cat "$count") == xxx ]] + +dev_cache_init 0 +rm -rf "$base" +dev_cache_run component ignored "$base" stage -- build_output uncached +[[ $(cat "$base/stage/file") == payload:uncached && $(cat "$count") == xxxx ]] + +echo 'development cache tests passed' diff --git a/os/mkosi/versions.env b/os/mkosi/versions.env new file mode 100644 index 000000000..4e2087b2c --- /dev/null +++ b/os/mkosi/versions.env @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# Every network input used by the experimental backend is immutable or hashed. +KERNEL_VERSION=7.1.4 +KERNEL_SHA256=1c63922a119675d38e3ae0f8f6ee07f15c41a786ab9ed66563749bb8c9a08e2e +DEBIAN_RELEASE=trixie +DEBIAN_SNAPSHOT=20260721T000000Z +MKOSI_VERSION=26 +RUST_TOOLCHAIN_VERSION=1.92.0 +RUSTC_TOOLCHAIN_SHA256=78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0 +CARGO_TOOLCHAIN_SHA256=e5e12be2c7126a7036c8adf573078a28b92611f5767cc9bd0a6f7c83081df103 +RUST_STD_TOOLCHAIN_SHA256=5f106805ed86ebf8df287039e53a45cf974391ef4d088c2760776b05b8e48b5d +GO_TOOLCHAIN_VERSION=1.22.2 +GO_TOOLCHAIN_SHA256=5901c52b7a78002aeff14a21f93e0f064f74ce1360fce51c6ee68cd471216a17 +NITRO_TPM_TOOLS_REVISION=d76d6eeebd4169b00a3c3af9858852d48f40e748 +DSTACK_VERSION=0.6.0 +OVMF_VARIANT=pre202505 +NVATTEST_VERSION=2026.06.09 +NVATTEST_REVISION=9d12801cea8a198ea0f29640dfaf8a4017c841c5 +NVIDIA_VERSION=595.58.03 +NVIDIA_RUN_SHA256=8c0d4f967b7932c4ab5714272aee8103392b0a702c92afa555176d36205829f9 +NVIDIA_FABRICMANAGER_SHA256=ea86951987ced2c2bbed1a9a9037e917dd2268add0ec484db9216e843d8b7de8 +NVIDIA_NSCQ_SHA256=1c008787111fc55a7fc1d3521a6d41c2523b8949354e5651eadb4498554e5469 +NVIDIA_CONTAINER_TOOLKIT_REVISION=c748619c592030519361274a2fdd43f2c2ced73a +LIBNVIDIA_CONTAINER_REVISION=889a3bb5408c195ed7897ba2cb8341c7d249672f +NERDCTL_VERSION=2.2.1 +NERDCTL_SHA256=34144de7f12756aa4b9dc42a907fd95b0c5eb82a63566a650ca10c8abe7a26a0 +CNI_VERSION=1.3.0 +CNI_SHA256=754a71ed60a4bd08726c3af705a7d55ee3df03122b12e389fdba4bea35d7dd7e +STARGZ_VERSION=0.18.2 +STARGZ_SHA256=515a3c3af0012f192ace31fb79e910597977c77227e976680aeaaef6e9ae50a9 +SYSBOX_VERSION=0.6.7 +SYSBOX_REVISION=3a69811f54f8f83264ebb36dcaf51708e80b9e84 +SYSBOX_RUNC_REVISION=c58eba1be027c762c495bc4eeba7c0984beda1ab +SYSBOX_FS_REVISION=6a8d71f54e7570e5297af89ff24ed3bafa61659f +SYSBOX_MGR_REVISION=aaeff6c5dc70c137e62166474a309ca5fc42d044 +SYSBOX_IPC_REVISION=f05151f4b4c1df63d7fd241577ca032905c1bd0e +SYSBOX_LIBS_REVISION=6faf00c74d45f7bdff0d1930fe8590c17a2d0a8b +SYSBOX_FUSE_REVISION=45cd9a3d884448418546d8eaa54ee7d772e576d6 +ZFS_VERSION=2.4.0 +ZFS_REVISION=743334913e5a5f60baf287bcc6d8a23515b02ac5 diff --git a/prek.toml b/prek.toml index 127d9c310..6c2c720b3 100644 --- a/prek.toml +++ b/prek.toml @@ -11,15 +11,15 @@ repo = "https://github.com/pre-commit/pre-commit-hooks" rev = "v5.0.0" hooks = [ - { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|tools/(dev-stack\.sh|vm-runner/)|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, - { id = "end-of-file-fixer", exclude = "^(os/yocto/|tools/(dev-stack\\.sh|vm-runner/))" }, + { id = "trailing-whitespace", args = ["--markdown-linebreak-ext=md"], exclude = '^(os/yocto/|os/mkosi/(patches/|components/[^/]+/patches/)|tools/(dev-stack\.sh|vm-runner/)|dstack/(dstack-util/tests/fixtures/key-provider-docker-compose\.yaml|gateway/src/main_service/snapshots/.*))' }, + { id = "end-of-file-fixer", exclude = "^(os/yocto/|os/mkosi/(patches/|components/[^/]+/patches/)|tools/(dev-stack\\.sh|vm-runner/))" }, { id = "check-yaml", args = ["--allow-multiple-documents"], exclude = "dstack/gateway/templates/" }, { id = "check-toml" }, { id = "check-json" }, { id = "check-merge-conflict" }, { id = "check-added-large-files", args = ["--maxkb=500"] }, { id = "check-symlinks" }, - { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^(os/yocto/|tools/(dev-stack\\.sh|vm-runner/))" }, + { id = "mixed-line-ending", args = ["--fix=lf"], exclude = "^(os/yocto/|os/mkosi/(patches/|components/[^/]+/patches/)|tools/(dev-stack\\.sh|vm-runner/))" }, ] # --- Rust: rustfmt ---