refactor(os): make mkosi components self-describing

Author: kvinwang

os/mkosi/README.md

@@ -60,11 +60,13 @@ DSTACK_DEV_CACHE_DIR="$HOME/.cache/dstack/mkosi-dev" \
 
 The cache covers dstack Rust, the container stack, Sysbox, nvattest, the
 kernel build tree, NVIDIA, ZFS and both OVMF variants. Its key conservatively
-includes all tracked and untracked mkosi/Yocto/dstack inputs, tool versions,
-architecture, flavor and `SOURCE_DATE_EPOCH`. `image` and `repro-check`
-unconditionally disable this development cache, even if the internal enable
-flag is inherited from the environment. Release artifacts, Debian rootfs,
-dm-verity data and measurements are never cached.
+includes the inputs, tools, packages and component dependencies declared by
+each file in `components/`, plus architecture, flavor and
+`SOURCE_DATE_EPOCH`. `build-components.sh` is intentionally only the ordered
+component list. Component install trees are merged with strict non-directory
+conflict detection. `image` and `repro-check` never pass the development-cache
+option. Release artifacts, Debian rootfs, dm-verity data and measurements are
+never cached.
 
 On a 16-job development host, a clean production work directory takes about
 17 minutes (measured 16m45s); allow 20--30 minutes with cold compiler and

os/mkosi/components/container-stack.sh

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=container-stack
+COMPONENT_CACHE_PATHS=(component-stages/container-stack)
+COMPONENT_ROOTFS_TREES=(component-stages/container-stack)
+COMPONENT_KERNEL_TREES=()
+
+component_cache_key() {
+    key_value "$NVIDIA_CONTAINER_TOOLKIT_REVISION" "$LIBNVIDIA_CONTAINER_REVISION" \
+      "$NERDCTL_VERSION" "$NERDCTL_SHA256" "$CNI_VERSION" "$CNI_SHA256" \
+      "$STARGZ_VERSION" "$STARGZ_SHA256"
+    key_file "$SELF/scripts/build-container-stack.sh" \
+      "$SELF/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch"
+    key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit \
+      os/yocto/layers/meta-dstack/recipes-containers
+    key_tools gcc go
+    key_packages libelf-dev libtirpc-dev
+}
+
+component_build() {
+    "$SELF/scripts/build-container-stack.sh" "$WORK/container-stack-build" \
+      "$WORK/component-stages/container-stack"
+}

os/mkosi/components/dstack-rust.sh

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=dstack-rust
+COMPONENT_CACHE_PATHS=(component-stages/dstack-rust)
+COMPONENT_ROOTFS_TREES=(component-stages/dstack-rust)
+COMPONENT_KERNEL_TREES=()
+
+component_cache_key() {
+    key_file "$SELF/scripts/stage-rootfs.sh"
+    key_tree dstack os/common/rootfs
+    key_tools rustc cargo
+}
+
+component_build() {
+    "$SELF/scripts/stage-rootfs.sh" "$WORK/component-stages/dstack-rust" "$FLAVOR"
+}

os/mkosi/components/kernel.sh

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=kernel
+COMPONENT_CACHE_PATHS=("linux-$KERNEL_VERSION" kernel-build component-stages/kernel)
+COMPONENT_ROOTFS_TREES=()
+COMPONENT_KERNEL_TREES=(component-stages/kernel)
+
+component_cache_key() {
+    key_value "$KERNEL_VERSION" "$KERNEL_SHA256"
+    key_file "$SELF/scripts/build-kernel.sh" "$SELF/kernel.config" \
+      "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch" \
+      "$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch"
+    key_tools gcc ld make pahole
+    key_packages binutils dwarves bc bison flex libssl-dev libelf-dev
+}
+
+component_build() {
+    "$SELF/scripts/build-kernel.sh" "$WORK" "$WORK/component-stages/kernel"
+}

os/mkosi/components/nvattest.sh

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=nvattest
+COMPONENT_CACHE_PATHS=(component-stages/nvattest)
+COMPONENT_ROOTFS_TREES=(component-stages/nvattest)
+COMPONENT_KERNEL_TREES=()
+
+component_cache_key() {
+    key_value "$NVATTEST_VERSION" "$NVATTEST_REVISION"
+    key_file "$SELF/scripts/build-nvattest.sh"
+    key_tree os/mkosi/patches/nvattest \
+      os/yocto/layers/meta-nvidia/recipes-graphics/nvattest
+    key_tools gcc cmake ninja rustc cargo
+    key_packages libssl-dev libcurl4-openssl-dev
+}
+
+component_build() {
+    "$SELF/scripts/build-nvattest.sh" "$WORK/nvattest-build" \
+      "$WORK/component-stages/nvattest"
+}

os/mkosi/components/nvidia.sh

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=nvidia
+COMPONENT_CACHE_PATHS=(component-stages/nvidia-root component-stages/nvidia-kernel)
+COMPONENT_ROOTFS_TREES=(component-stages/nvidia-root)
+COMPONENT_KERNEL_TREES=(component-stages/nvidia-kernel)
+
+component_cache_key() {
+    key_dependency kernel
+    key_value "$NVIDIA_VERSION" "$NVIDIA_RUN_SHA256" \
+      "$NVIDIA_FABRICMANAGER_SHA256" "$NVIDIA_NSCQ_SHA256"
+    key_file "$SELF/scripts/build-nvidia.sh" \
+      "$SELF/patches/nvidia/0001-linux-7.1-drop-legacy-of-gpio-api.patch"
+    key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvidia
+    key_tools gcc make
+}
+
+component_build() {
+    "$SELF/scripts/build-nvidia.sh" "$WORK/nvidia-build" \
+      "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" \
+      "$WORK/component-stages/nvidia-root" "$WORK/component-stages/nvidia-kernel"
+}
+
+component_prepare_outputs() {
+    find "$WORK/component-stages/nvidia-kernel/usr/lib/modules" -maxdepth 2 \
+      -type f -name 'modules.*' -delete
+}

os/mkosi/components/ovmf.sh

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=ovmf
+COMPONENT_CACHE_PATHS=(component-stages/ovmf)
+COMPONENT_ROOTFS_TREES=()
+COMPONENT_KERNEL_TREES=(component-stages/ovmf)
+
+component_cache_key() {
+    key_file "$SELF/scripts/build-ovmf.sh" \
+      "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-ovmf/dstack-ovmf/0004-Reproduciable.patch"
+    key_tools gcc make python3
+    key_packages nasm acpica-tools uuid-dev
+}
+
+component_build() {
+    mkdir -p "$WORK/component-stages/ovmf"
+    "$SELF/scripts/build-ovmf.sh" "$WORK/ovmf-build" \
+      "$WORK/component-stages/ovmf/ovmf.fd" \
+      "$WORK/component-stages/ovmf/ovmf-sev.fd"
+}

os/mkosi/components/sysbox.sh

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=sysbox
+COMPONENT_CACHE_PATHS=(component-stages/sysbox)
+COMPONENT_ROOTFS_TREES=(component-stages/sysbox)
+COMPONENT_KERNEL_TREES=()
+
+component_cache_key() {
+    key_value "$SYSBOX_VERSION" "$SYSBOX_REVISION" "$SYSBOX_RUNC_REVISION" \
+      "$SYSBOX_FS_REVISION" "$SYSBOX_MGR_REVISION" "$SYSBOX_IPC_REVISION" \
+      "$SYSBOX_LIBS_REVISION" "$SYSBOX_FUSE_REVISION"
+    key_file "$SELF/scripts/build-sysbox.sh"
+    key_tree os/yocto/layers/meta-dstack/recipes-core/dstack-sysbox
+    key_tools go
+    key_packages libseccomp-dev
+}
+
+component_build() {
+    "$SELF/scripts/build-sysbox.sh" "$WORK/sysbox-build" \
+      "$WORK/component-stages/sysbox"
+}

os/mkosi/components/zfs.sh

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# shellcheck shell=bash disable=SC2034
+COMPONENT_NAME=zfs
+COMPONENT_CACHE_PATHS=(component-stages/zfs-root component-stages/zfs-kernel)
+COMPONENT_ROOTFS_TREES=(component-stages/zfs-root)
+COMPONENT_KERNEL_TREES=(component-stages/zfs-kernel)
+
+component_cache_key() {
+    key_dependency kernel
+    key_value "$ZFS_VERSION" "$ZFS_REVISION"
+    key_file "$SELF/scripts/build-zfs.sh" \
+      "$SELF/patches/zfs/0001-linux-6.19-7.1-compat.patch" \
+      "$ROOT/os/yocto/layers/meta-dstack/recipes-core/dstack-zfs/dstack-zfs/0001-Define-strndupa-if-it-does-not-exist.patch"
+    key_tools gcc make autoconf automake
+    key_packages libssl-dev libelf-dev zlib1g-dev libtool uuid-dev
+}
+
+component_build() {
+    "$SELF/scripts/build-zfs.sh" "$WORK/zfs-build" \
+      "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" \
+      "$WORK/component-stages/zfs-root" "$WORK/component-stages/zfs-kernel"
+}
+
+component_prepare_outputs() {
+    find "$WORK/component-stages/zfs-kernel/usr/lib/modules" -maxdepth 2 \
+      -type f -name 'modules.*' -delete
+}

os/mkosi/scripts/build-components.sh

@@ -1,65 +1,35 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
-# Build and compose every non-Debian guest component. Cache-aware staging is
-# kept here so the release pipeline in build.sh remains cache-agnostic.
 set -euo pipefail
 ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
 SELF="$ROOT/os/mkosi"
 # shellcheck source=/dev/null
 source "$SELF/versions.env"
 # shellcheck source=/dev/null
 source "$SELF/scripts/dev-cache.sh"
+# shellcheck source=/dev/null
+source "$SELF/scripts/component-framework.sh"
 
 use_cache=0
 if [[ ${1:-} == --dev-cache ]]; then use_cache=1; shift; fi
-WORK=$(realpath -m "${1:?work directory required}")
-STAGE=$(realpath -m "${2:?rootfs staging tree required}")
-KSTAGE=$(realpath -m "${3:?kernel staging tree required}")
-FLAVOR=${4:?flavor required}
-COMPONENTS="$WORK/component-stages"
-mkdir -p "$STAGE" "$KSTAGE" "$COMPONENTS"
-dev_cache_init "$use_cache" "$ROOT" "$FLAVOR" "${SOURCE_DATE_EPOCH:?}"
-
-merge() { cp -a "$1/." "$2/"; }
-
-root="$COMPONENTS/dstack-rust"
-dev_cache_run dstack-rust "$WORK" component-stages/dstack-rust -- \
-  "$SELF/scripts/stage-rootfs.sh" "$root" "$FLAVOR"
-merge "$root" "$STAGE"
-
-root="$COMPONENTS/container-stack"
-dev_cache_run container-stack "$WORK" component-stages/container-stack -- \
-  "$SELF/scripts/build-container-stack.sh" "$WORK/container-stack-build" "$root"
-merge "$root" "$STAGE"
-
-root="$COMPONENTS/sysbox"
-dev_cache_run sysbox "$WORK" component-stages/sysbox -- \
-  "$SELF/scripts/build-sysbox.sh" "$WORK/sysbox-build" "$root"
-merge "$root" "$STAGE"
-
-root="$COMPONENTS/nvattest"
-dev_cache_run nvattest "$WORK" component-stages/nvattest -- \
-  "$SELF/scripts/build-nvattest.sh" "$WORK/nvattest-build" "$root"
-merge "$root" "$STAGE"
-
-dev_cache_run kernel "$WORK" "linux-$KERNEL_VERSION" kernel-build kernel-stage -- \
-  "$SELF/scripts/build-kernel.sh" "$WORK" "$KSTAGE"
-
-root="$COMPONENTS/nvidia-root" modules="$COMPONENTS/nvidia-kernel"
-dev_cache_run nvidia "$WORK" component-stages/nvidia-root component-stages/nvidia-kernel -- \
-  "$SELF/scripts/build-nvidia.sh" "$WORK/nvidia-build" \
-    "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" "$root" "$modules"
-merge "$root" "$STAGE"; merge "$modules" "$KSTAGE"
-
-root="$COMPONENTS/zfs-root" modules="$COMPONENTS/zfs-kernel"
-dev_cache_run zfs "$WORK" component-stages/zfs-root component-stages/zfs-kernel -- \
-  "$SELF/scripts/build-zfs.sh" "$WORK/zfs-build" \
-    "$WORK/linux-$KERNEL_VERSION" "$WORK/kernel-build" "$root" "$modules"
-merge "$root" "$STAGE"; merge "$modules" "$KSTAGE"
-
-root="$COMPONENTS/ovmf"; mkdir -p "$root"
-dev_cache_run ovmf "$WORK" component-stages/ovmf -- \
-  "$SELF/scripts/build-ovmf.sh" "$WORK/ovmf-build" \
-    "$root/ovmf.fd" "$root/ovmf-sev.fd"
-install -Dm0644 "$root/ovmf.fd" "$KSTAGE/ovmf.fd"
-install -Dm0644 "$root/ovmf-sev.fd" "$KSTAGE/ovmf-sev.fd"
+work=${1:?work directory required}
+stage=${2:?rootfs staging tree required}
+kstage=${3:?kernel staging tree required}
+flavor=${4:?flavor required}
+component_framework_init "$use_cache" "$ROOT" "$SELF" \
+  "$(realpath -m "$work")" "$(realpath -m "$stage")" \
+  "$(realpath -m "$kstage")" "$flavor"
+
+components=(
+  dstack-rust
+  container-stack
+  sysbox
+  nvattest
+  kernel
+  nvidia
+  zfs
+  ovmf
+)
+
+for component in "${components[@]}"; do component_run "$component"; done
+component_assemble