Skip to content

Commit 8500c94

Browse files
committed
refactor(os): colocate mkosi component definitions
1 parent 727d48b commit 8500c94

26 files changed

Lines changed: 67 additions & 53 deletions

os/mkosi/README.md

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,21 @@ DSTACK_DEV_CACHE_DIR="$HOME/.cache/dstack/mkosi-dev" \
6161
The cache covers dstack Rust, the container stack, Sysbox, nvattest, the
6262
kernel build tree, NVIDIA, ZFS and both OVMF variants. Its key conservatively
6363
includes the inputs, tools, packages and component dependencies declared by
64-
each file in `components/`, plus architecture, flavor and
64+
each descriptor in `components/<name>/<name>.sh`, plus architecture, flavor and
6565
`SOURCE_DATE_EPOCH`. `build-components.sh` is intentionally only the ordered
6666
component list. Component install trees are merged with strict non-directory
6767
conflict detection. `image` and `repro-check` never pass the development-cache
6868
option. Release artifacts, Debian rootfs, dm-verity data and measurements are
6969
never cached.
7070

71+
mkosi's `Incremental=`, `CacheDirectory=` and `BuildDirectory=` cover
72+
whole-image/rootfs and persistent-work-directory reuse; they do not provide
73+
independently keyed output trees for components or reject file collisions when
74+
those trees are installed. The small component layer only supplies those two
75+
missing policies. Source download, build, cache-key inputs and output ownership
76+
remain together under `components/<name>/`; production builds bypass the layer's
77+
archive cache completely.
78+
7179
On a 16-job development host, a clean production work directory takes about
7280
17 minutes (measured 16m45s); allow 20--30 minutes with cold compiler and
7381
network caches. `repro-check` performs two such builds sequentially.
@@ -84,7 +92,7 @@ OVMF and UKI. Debian supplies the base userspace while the parity checker
8492
requires the Yocto-visible binaries, services, configuration, kernel modules
8593
and production/development separation before assembly is allowed to proceed.
8694

87-
The firmware is not Debian's generic OVMF: `build-ovmf.sh` builds the same
95+
The firmware is not Debian's generic OVMF: `components/ovmf/ovmf-build.sh` builds the same
8896
EDK2 stable-202502 revision and `pre202505` TDX measurement layout selected by
8997
the Yocto recipe. A generic OVMF cannot be substituted because `dstack-mr`
9098
would produce invalid or unparseable TDX measurement material.

os/mkosi/scripts/build-container-stack.sh renamed to os/mkosi/components/container-stack/container-stack-build.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# SPDX-License-Identifier: Apache-2.0
33
set -euo pipefail
4-
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
4+
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
55
SELF="$ROOT/os/mkosi"
66
# shellcheck source=/dev/null
77
source "$SELF/versions.env"
@@ -30,7 +30,7 @@ fetch_sha256() {
3030
lnc="$BUILD_DIR/libnvidia-container"
3131
checkout https://github.com/NVIDIA/libnvidia-container.git "$LIBNVIDIA_CONTAINER_REVISION" "$lnc"
3232
patch -d "$lnc" -p1 --fuzz=0 < \
33-
"$SELF/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch"
33+
"$SELF/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch"
3434
export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$BUILD_DIR=/usr/src/container-stack -fmacro-prefix-map=$BUILD_DIR=/usr/src/container-stack"
3535
export LDFLAGS="${LDFLAGS:-} -Wl,--build-id=none"
3636
export CGO_CFLAGS="$CFLAGS"

os/mkosi/components/container-stack.sh renamed to os/mkosi/components/container-stack/container-stack.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,15 @@ component_cache_key() {
99
key_value "$NVIDIA_CONTAINER_TOOLKIT_REVISION" "$LIBNVIDIA_CONTAINER_REVISION" \
1010
"$NERDCTL_VERSION" "$NERDCTL_SHA256" "$CNI_VERSION" "$CNI_SHA256" \
1111
"$STARGZ_VERSION" "$STARGZ_SHA256"
12-
key_file "$SELF/scripts/build-container-stack.sh" \
13-
"$SELF/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch"
12+
key_file "$COMPONENT_PATH/container-stack-build.sh" \
13+
"$COMPONENT_PATH/patches/0001-omit-prefix-map-from-build-flags.patch"
1414
key_tree os/yocto/layers/meta-nvidia/recipes-graphics/nvidia-container-toolkit \
1515
os/yocto/layers/meta-dstack/recipes-containers
1616
key_tools gcc go
1717
key_packages libelf-dev libtirpc-dev
1818
}
1919

2020
component_build() {
21-
"$SELF/scripts/build-container-stack.sh" "$WORK/container-stack-build" \
21+
"$COMPONENT_PATH/container-stack-build.sh" "$WORK/container-stack-build" \
2222
"$WORK/component-stages/container-stack"
2323
}

os/mkosi/patches/libnvidia-container/0001-omit-prefix-map-from-build-flags.patch renamed to os/mkosi/components/container-stack/patches/0001-omit-prefix-map-from-build-flags.patch

File renamed without changes.

os/mkosi/scripts/stage-rootfs.sh renamed to os/mkosi/components/dstack-rust/dstack-rust-build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# SPDX-License-Identifier: Apache-2.0
33
set -euo pipefail
4-
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
4+
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
55
DEST=${1:?staging tree required}
66
DEST=$(realpath -m "$DEST")
77
FLAVOR=${2:-prod}

os/mkosi/components/dstack-rust.sh renamed to os/mkosi/components/dstack-rust/dstack-rust.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ COMPONENT_ROOTFS_TREES=(component-stages/dstack-rust)
66
COMPONENT_KERNEL_TREES=()
77

88
component_cache_key() {
9-
key_file "$SELF/scripts/stage-rootfs.sh"
9+
key_file "$COMPONENT_PATH/dstack-rust-build.sh"
1010
key_tree dstack os/common/rootfs
1111
key_tools rustc cargo
1212
}
1313

1414
component_build() {
15-
"$SELF/scripts/stage-rootfs.sh" "$WORK/component-stages/dstack-rust" "$FLAVOR"
15+
"$COMPONENT_PATH/dstack-rust-build.sh" "$WORK/component-stages/dstack-rust" "$FLAVOR"
1616
}

os/mkosi/scripts/build-kernel.sh renamed to os/mkosi/components/kernel/kernel-build.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# SPDX-License-Identifier: Apache-2.0
33
set -euo pipefail
4-
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
4+
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
55
MKOSI_DIR="$ROOT/os/mkosi"
66
# shellcheck source=/dev/null
77
source "$MKOSI_DIR/versions.env"
@@ -58,7 +58,7 @@ done
5858

5959
make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" x86_64_defconfig
6060
"$src/scripts/kconfig/merge_config.sh" -m -O "$BUILD_DIR/kernel-build" \
61-
"$BUILD_DIR/kernel-build/.config" "$MKOSI_DIR/kernel.config"
61+
"$BUILD_DIR/kernel-build/.config" "$MKOSI_DIR/components/kernel/kernel.config"
6262
make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" olddefconfig
6363
"$MKOSI_DIR/scripts/check-kernel-config.sh" "$BUILD_DIR/kernel-build/.config"
6464
make -C "$src" O="$BUILD_DIR/kernel-build" PAHOLE="$pahole_wrapper" -j"$JOBS" bzImage modules
Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ COMPONENT_KERNEL_TREES=(component-stages/kernel)
77

88
component_cache_key() {
99
key_value "$KERNEL_VERSION" "$KERNEL_SHA256"
10-
key_file "$SELF/scripts/build-kernel.sh" "$SELF/kernel.config" \
10+
key_file "$COMPONENT_PATH/kernel-build.sh" "$COMPONENT_PATH/kernel.config" \
1111
"$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0001-x86-tdx-select-dma-direct-remap.patch" \
1212
"$ROOT/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/0002-acpi-sandbox-block-aml-systemmemory-ram-access.patch"
1313
key_tools gcc ld make pahole
1414
key_packages binutils dwarves bc bison flex libssl-dev libelf-dev
1515
}
1616

1717
component_build() {
18-
"$SELF/scripts/build-kernel.sh" "$WORK" "$WORK/component-stages/kernel"
18+
"$COMPONENT_PATH/kernel-build.sh" "$WORK" "$WORK/component-stages/kernel"
1919
}

os/mkosi/scripts/build-nvattest.sh renamed to os/mkosi/components/nvattest/nvattest-build.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
# SPDX-License-Identifier: Apache-2.0
33
set -euo pipefail
4-
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
4+
ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
55
SELF="$ROOT/os/mkosi"
66
# shellcheck source=/dev/null
77
source "$SELF/versions.env"
@@ -35,7 +35,7 @@ cp -a "$checkout" "$src"
3535
patch -d "$src" -p1 --fuzz=0 < \
3636
"$ROOT/os/yocto/layers/meta-nvidia/recipes-graphics/nvattest/files/0001-validate-ocsp-response-freshness.patch"
3737
patch -d "$src" -p1 --fuzz=0 < \
38-
"$SELF/patches/nvattest/0001-pin-fetchcontent-inputs.patch"
38+
"$SELF/components/nvattest/patches/0001-pin-fetchcontent-inputs.patch"
3939

4040
rm -rf "$build"
4141
export CFLAGS="${CFLAGS:-} -O2 -g0 -ffile-prefix-map=$canonical=/usr/src/nvattest -fmacro-prefix-map=$canonical=/usr/src/nvattest"

0 commit comments

Comments
 (0)