Skip to content

Commit e9a8ffc

Browse files
kvinwangkvinwang
committed
feat(os): add opt-in mkosi development cache
1 parent a158ce3 commit e9a8ffc

5 files changed

Lines changed: 198 additions & 15 deletions

File tree

‎os/mkosi/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,21 @@ qemu-system-x86_64 -machine q35 -m 2G -nographic \
5151
-drive if=virtio,format=raw,file=os/mkosi/build/out/prod/dstack-0.6.0/disk.raw
5252
```
5353

54+
For local iteration only, `dev-image` enables a component-output cache:
55+
56+
```sh
57+
DSTACK_DEV_CACHE_DIR="$HOME/.cache/dstack/mkosi-dev" \
58+
./os/mkosi/build.sh dev-image "$PWD/os/mkosi/build-dev"
59+
```
60+
61+
The cache covers dstack Rust, the container stack, Sysbox, nvattest, the
62+
kernel build tree, NVIDIA, ZFS and both OVMF variants. Its key conservatively
63+
includes all tracked and untracked mkosi/Yocto/dstack inputs, tool versions,
64+
architecture, flavor and `SOURCE_DATE_EPOCH`. `image` and `repro-check`
65+
unconditionally disable this development cache, even if the internal enable
66+
flag is inherited from the environment. Release artifacts, Debian rootfs,
67+
dm-verity data and measurements are never cached.
68+
5469
On a 16-job development host, a clean production work directory takes about
5570
17 minutes (measured 16m45s); allow 20--30 minutes with cold compiler and
5671
network caches. `repro-check` performs two such builds sequentially.

‎os/mkosi/build.sh‎

Lines changed: 85 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,14 @@ action=${1:-image}
99
BUILD_DIR=${2:-$SELF/build}
1010
BUILD_DIR=$(realpath -m "$BUILD_DIR")
1111
case "$action" in
12-
image|repro-check|lint) ;;
13-
*) echo "Usage: $0 {image|repro-check|lint} [build-dir]" >&2; exit 2 ;;
12+
image|dev-image|repro-check|lint) ;;
13+
*) echo "Usage: $0 {image|dev-image|repro-check|lint} [build-dir]" >&2; exit 2 ;;
1414
esac
1515
if [[ $action == lint ]]; then exec "$SELF/tests/acceptance.sh"; fi
16+
unset DSTACK_DEV_CACHE_ACTIVE
17+
[[ $action == dev-image ]] && export DSTACK_DEV_CACHE_ACTIVE=1
18+
# shellcheck source=/dev/null
19+
source "$SELF/scripts/dev-cache.sh"
1620
command -v mkosi >/dev/null || { echo 'mkosi >= 26 is required' >&2; exit 1; }
1721
actual=$(mkosi --version | awk '{print $2}' | cut -d. -f1)
1822
(( actual >= MKOSI_MIN_VERSION )) || { echo "mkosi >= $MKOSI_MIN_VERSION required" >&2; exit 1; }
@@ -22,18 +26,84 @@ export TZ=UTC LC_ALL=C
2226
build_one() {
2327
local out=$1 work=$2 flavor=$3
2428
local stage="$work/rootfs-stage" kstage="$work/kernel-stage" tree="$work/rootfs"
25-
rm -rf "$work" "$out"; mkdir -p "$stage" "$kstage" "$out"
26-
"$SELF/scripts/stage-rootfs.sh" "$stage" "$flavor"
27-
"$SELF/scripts/build-container-stack.sh" "$work/container-stack-build" "$stage"
28-
"$SELF/scripts/build-sysbox.sh" "$work/sysbox-build" "$stage"
29-
"$SELF/scripts/build-nvattest.sh" "$work/nvattest-build" "$stage"
30-
"$SELF/scripts/build-kernel.sh" "$work" "$kstage"
31-
"$SELF/scripts/build-nvidia.sh" "$work/nvidia-build" \
32-
"$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$stage" "$kstage"
33-
"$SELF/scripts/build-zfs.sh" "$work/zfs-build" \
34-
"$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$stage" "$kstage"
35-
"$SELF/scripts/build-ovmf.sh" "$work/ovmf-build" \
36-
"$kstage/ovmf.fd" "$kstage/ovmf-sev.fd"
29+
local components="$work/component-stages" source_fingerprint=disabled tool_fingerprint=disabled
30+
rm -rf "$work" "$out"; mkdir -p "$stage" "$kstage" "$out" "$components"
31+
if [[ ${DSTACK_DEV_CACHE_ACTIVE:-0} == 1 ]]; then
32+
source_fingerprint=$(
33+
git -C "$ROOT" ls-files -z --cached --others --exclude-standard -- \
34+
dstack os/common os/mkosi os/yocto/layers/meta-dstack \
35+
os/yocto/layers/meta-nvidia \
36+
| sort -z | while IFS= read -r -d '' input; do
37+
if [[ -f $ROOT/$input ]]; then
38+
sha256sum "$ROOT/$input"
39+
elif [[ -d $ROOT/$input ]]; then
40+
printf 'gitlink %s %s\n' "$input" \
41+
"$(git -C "$ROOT/$input" rev-parse HEAD)"
42+
else
43+
printf 'missing %s\n' "$input"
44+
fi
45+
done | sha256sum | cut -d' ' -f1
46+
)
47+
tool_fingerprint=$({
48+
gcc --version | head -1; ld --version | head -1; go version
49+
rustc --version; cargo --version; cmake --version | head -1
50+
make --version | head -1; pahole --version; tar --version | head -1
51+
} | sha256sum | cut -d' ' -f1)
52+
fi
53+
component_key() {
54+
dev_cache_key "$1" "$flavor" "$source_fingerprint" \
55+
"$tool_fingerprint" "$SOURCE_DATE_EPOCH" "$(uname -m)"
56+
}
57+
merge_stage() { cp -a "$1/." "$2/"; }
58+
59+
base_stage="$components/dstack-rust"
60+
dev_cache_run dstack-rust "$(component_key dstack-rust)" "$work" 1 \
61+
component-stages/dstack-rust -- \
62+
"$SELF/scripts/stage-rootfs.sh" "$base_stage" "$flavor"
63+
merge_stage "$base_stage" "$stage"
64+
65+
container_stage="$components/container-stack"
66+
dev_cache_run container-stack "$(component_key container-stack)" "$work" 1 \
67+
component-stages/container-stack -- \
68+
"$SELF/scripts/build-container-stack.sh" "$work/container-stack-build" "$container_stage"
69+
merge_stage "$container_stage" "$stage"
70+
71+
sysbox_stage="$components/sysbox"
72+
dev_cache_run sysbox "$(component_key sysbox)" "$work" 1 \
73+
component-stages/sysbox -- \
74+
"$SELF/scripts/build-sysbox.sh" "$work/sysbox-build" "$sysbox_stage"
75+
merge_stage "$sysbox_stage" "$stage"
76+
77+
nvattest_stage="$components/nvattest"
78+
dev_cache_run nvattest "$(component_key nvattest)" "$work" 1 \
79+
component-stages/nvattest -- \
80+
"$SELF/scripts/build-nvattest.sh" "$work/nvattest-build" "$nvattest_stage"
81+
merge_stage "$nvattest_stage" "$stage"
82+
83+
dev_cache_run kernel "$(component_key kernel)" "$work" 3 \
84+
"linux-$KERNEL_VERSION" kernel-build kernel-stage -- \
85+
"$SELF/scripts/build-kernel.sh" "$work" "$kstage"
86+
87+
nvidia_root="$components/nvidia-root" nvidia_kernel="$components/nvidia-kernel"
88+
dev_cache_run nvidia "$(component_key nvidia)" "$work" 2 \
89+
component-stages/nvidia-root component-stages/nvidia-kernel -- \
90+
"$SELF/scripts/build-nvidia.sh" "$work/nvidia-build" \
91+
"$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$nvidia_root" "$nvidia_kernel"
92+
merge_stage "$nvidia_root" "$stage"; merge_stage "$nvidia_kernel" "$kstage"
93+
94+
zfs_root="$components/zfs-root" zfs_kernel="$components/zfs-kernel"
95+
dev_cache_run zfs "$(component_key zfs)" "$work" 2 \
96+
component-stages/zfs-root component-stages/zfs-kernel -- \
97+
"$SELF/scripts/build-zfs.sh" "$work/zfs-build" \
98+
"$work/linux-$KERNEL_VERSION" "$work/kernel-build" "$zfs_root" "$zfs_kernel"
99+
merge_stage "$zfs_root" "$stage"; merge_stage "$zfs_kernel" "$kstage"
100+
101+
ovmf_stage="$components/ovmf"; mkdir -p "$ovmf_stage"
102+
dev_cache_run ovmf "$(component_key ovmf)" "$work" 1 component-stages/ovmf -- \
103+
"$SELF/scripts/build-ovmf.sh" "$work/ovmf-build" \
104+
"$ovmf_stage/ovmf.fd" "$ovmf_stage/ovmf-sev.fd"
105+
install -Dm0644 "$ovmf_stage/ovmf.fd" "$kstage/ovmf.fd"
106+
install -Dm0644 "$ovmf_stage/ovmf-sev.fd" "$kstage/ovmf-sev.fd"
37107
# ExtraTrees is copied over Debian's usr-merged root where /bin, /sbin and
38108
# /lib are symlinks. Normalize build systems (notably OpenZFS) that install
39109
# into the legacy physical directories before handing the tree to mkosi.
@@ -84,7 +154,7 @@ EOF
84154
"$SELF/tests/check-output.sh" "$out/$release_name-$DSTACK_VERSION" "$flavor"
85155
}
86156

87-
if [[ $action == image ]]; then
157+
if [[ $action == image || $action == dev-image ]]; then
88158
for flavor in $FLAVORS; do build_one "$BUILD_DIR/out/$flavor" "$BUILD_DIR/work-$flavor" "$flavor"; done
89159
exit
90160
fi

‎os/mkosi/scripts/dev-cache.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
# Development-only, local component output cache. Production and reproducible
4+
# builds never set DSTACK_DEV_CACHE_ACTIVE and therefore bypass this file.
5+
6+
dev_cache_key() {
7+
printf '%s\0' "mkosi-dev-cache-v1" "$@" | sha256sum | cut -d' ' -f1
8+
}
9+
10+
dev_cache_run() {
11+
local component=$1 key=$2 base=$3 output_count=$4
12+
shift 4
13+
local outputs=("${@:1:output_count}")
14+
shift "$output_count"
15+
[[ $1 == -- ]]; shift
16+
17+
if [[ ${DSTACK_DEV_CACHE_ACTIVE:-0} != 1 ]]; then
18+
"$@"
19+
return
20+
fi
21+
22+
local cache_dir=${DSTACK_DEV_CACHE_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/dstack/mkosi-dev}
23+
local dir="$cache_dir/$component"
24+
local archive="$dir/$key.tar.zst"
25+
local checksum="$archive.sha256" lock="$dir/$key.lock" tmp
26+
mkdir -p "$dir" "$base"
27+
exec {cache_lock}>"$lock"
28+
flock "$cache_lock"
29+
30+
if [[ -f $archive && -f $checksum ]] &&
31+
(cd "$dir" && sha256sum --check --status "${checksum##*/}"); then
32+
tar --zstd -xf "$archive" -C "$base"
33+
echo "development cache hit: $component"
34+
flock -u "$cache_lock"
35+
exec {cache_lock}>&-
36+
return
37+
fi
38+
39+
# A partial or corrupt entry is only a cache miss; it is never trusted.
40+
rm -f "$archive" "$checksum"
41+
echo "development cache miss: $component"
42+
"$@"
43+
44+
tmp=$(mktemp "$dir/.${key}.XXXXXX.tar.zst")
45+
tar --zstd -cf "$tmp" -C "$base" "${outputs[@]}"
46+
mv "$tmp" "$archive"
47+
(cd "$dir" && sha256sum "${archive##*/}" > "${checksum##*/}.tmp")
48+
mv "$checksum.tmp" "$checksum"
49+
flock -u "$cache_lock"
50+
exec {cache_lock}>&-
51+
}

‎os/mkosi/tests/acceptance.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,7 @@ grep -q -- '--fuzz=0' "$D/scripts/build-zfs.sh"
4242
python3 -m json.tool "$D/parity.json" >/dev/null
4343
grep -q 'rootfs.img.parted.verity' "$D/../image/assemble.sh"
4444
bash -n "$D"/*.sh "$D"/scripts/*.sh "$D"/tests/*.sh
45+
grep -q '^unset DSTACK_DEV_CACHE_ACTIVE$' "$D/build.sh"
46+
grep -Fq "[[ \$action == dev-image ]] && export DSTACK_DEV_CACHE_ACTIVE=1" "$D/build.sh"
47+
"$D/tests/test-dev-cache.sh"
4548
echo 'mkosi static acceptance passed'

‎os/mkosi/tests/test-dev-cache.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
set -euo pipefail
4+
D=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
5+
# shellcheck source=/dev/null
6+
source "$D/scripts/dev-cache.sh"
7+
8+
tmp=$(mktemp -d)
9+
trap 'rm -rf "$tmp"' EXIT
10+
export DSTACK_DEV_CACHE_ACTIVE=1 DSTACK_DEV_CACHE_DIR="$tmp/cache"
11+
base="$tmp/work"
12+
count="$tmp/count"
13+
14+
build_output() {
15+
mkdir -p "$base/stage"
16+
printf 'payload:%s\n' "$1" > "$base/stage/file"
17+
printf x >> "$count"
18+
}
19+
20+
key=$(dev_cache_key component input-v1)
21+
dev_cache_run component "$key" "$base" 1 stage -- build_output v1
22+
[[ $(cat "$base/stage/file") == payload:v1 && $(cat "$count") == x ]]
23+
24+
rm -rf "$base"
25+
dev_cache_run component "$key" "$base" 1 stage -- build_output wrong
26+
[[ $(cat "$base/stage/file") == payload:v1 && $(cat "$count") == x ]]
27+
28+
rm -rf "$base"
29+
key2=$(dev_cache_key component input-v2)
30+
dev_cache_run component "$key2" "$base" 1 stage -- build_output v2
31+
[[ $(cat "$base/stage/file") == payload:v2 && $(cat "$count") == xx ]]
32+
33+
rm -rf "$base"
34+
archive=$(find "$DSTACK_DEV_CACHE_DIR/component" -name "$key2.tar.zst")
35+
printf corrupt >> "$archive"
36+
dev_cache_run component "$key2" "$base" 1 stage -- build_output rebuilt
37+
[[ $(cat "$base/stage/file") == payload:rebuilt && $(cat "$count") == xxx ]]
38+
39+
unset DSTACK_DEV_CACHE_ACTIVE
40+
rm -rf "$base"
41+
dev_cache_run component ignored "$base" 1 stage -- build_output uncached
42+
[[ $(cat "$base/stage/file") == payload:uncached && $(cat "$count") == xxxx ]]
43+
44+
echo 'development cache tests passed'

0 commit comments

Comments
 (0)