Skip to content

Commit 5f1d893

Browse files
committed
ci: split distro-qemu into build-initramfs + run-qemu jobs
The initramfs (test binaries + busybox + shared libs) is arch-specific but kernel-independent. Split into: - build-distro-qemu-initramfs: builds once per arch, uploads artifact - distro-qemu-tests: downloads initramfs + kernel, runs QEMU only This avoids rebuilding Go test binaries and the rootfs for every kernel version (was ~46s per job on amd64: 27s go build + 15s rootfs + 4s initramfs). Each test job now only needs QEMU + kernel download. The build-and-run.sh wrapper is kept for local development convenience.
1 parent c0fd898 commit 5f1d893

4 files changed

Lines changed: 319 additions & 260 deletions

File tree

.github/workflows/unit-test-on-pull-request.yml

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -233,10 +233,50 @@ jobs:
233233
uname -a
234234
sudo go test ./interpreter/... -v -run "TestIntegration/(node-local-nightly|node-latest)"
235235
236+
build-distro-qemu-initramfs:
237+
name: Build distro QEMU initramfs (${{ matrix.target_arch }})
238+
runs-on: ${{ matrix.target_arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
239+
timeout-minutes: 10
240+
strategy:
241+
matrix:
242+
target_arch: [amd64, arm64]
243+
steps:
244+
- name: Clone code
245+
uses: actions/checkout@v4
246+
- name: Set up Go
247+
uses: actions/setup-go@v5
248+
with:
249+
go-version-file: go.mod
250+
cache-dependency-path: go.sum
251+
- name: Install dependencies
252+
run: |
253+
sudo apt-get update -y
254+
sudo apt-get install -y --no-install-recommends busybox-static systemtap-sdt-dev
255+
- name: Cache parcagpu library
256+
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
257+
with:
258+
path: test/distro-qemu/parcagpu-lib
259+
key: parcagpu-${{ matrix.target_arch }}
260+
- name: Build initramfs
261+
run: |
262+
cd test/distro-qemu
263+
case "${{ matrix.target_arch }}" in
264+
amd64) export QEMU_ARCH=x86_64;;
265+
arm64) export QEMU_ARCH=aarch64;;
266+
*) echo >&2 "bug: bad arch selected"; exit 1;;
267+
esac
268+
./build-initramfs.sh initramfs-${{ matrix.target_arch }}.gz
269+
- name: Upload initramfs
270+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
271+
with:
272+
name: distro-qemu-initramfs-${{ matrix.target_arch }}
273+
path: test/distro-qemu/initramfs-${{ matrix.target_arch }}.gz
274+
236275
distro-qemu-tests:
237276
name: Distro QEMU tests (${{ matrix.kernel }} ${{ matrix.target_arch }})
238277
runs-on: ${{ matrix.target_arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
239-
timeout-minutes: 15
278+
needs: build-distro-qemu-initramfs
279+
timeout-minutes: 10
240280
strategy:
241281
matrix:
242282
include:
@@ -261,11 +301,6 @@ jobs:
261301
steps:
262302
- name: Clone code
263303
uses: actions/checkout@v4
264-
- name: Set up Go
265-
uses: actions/setup-go@v5
266-
with:
267-
go-version-file: go.mod
268-
cache-dependency-path: go.sum
269304
- name: Install dependencies
270305
run: |
271306
sudo tee /etc/apt/sources.list.d/ubuntu.sources <<EOF > /dev/null
@@ -292,12 +327,11 @@ jobs:
292327
arm64) sudo apt-get -y install qemu-system-arm ipxe-qemu gcc-aarch64-linux-gnu libc6-arm64-cross libc6:arm64 binutils-aarch64-linux-gnu musl-dev:arm64 systemtap-sdt-dev:arm64;;
293328
*) echo >&2 "bug: bad arch selected"; exit 1;;
294329
esac
295-
sudo apt-get install -y --no-install-recommends busybox-static systemtap-sdt-dev
296-
- name: Cache parcagpu library
297-
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
330+
- name: Fetch initramfs
331+
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
298332
with:
299-
path: test/distro-qemu/parcagpu-lib
300-
key: parcagpu-${{ matrix.target_arch }}
333+
name: distro-qemu-initramfs-${{ matrix.target_arch }}
334+
path: test/distro-qemu
301335
- name: Cache kernel image
302336
uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
303337
with:
@@ -312,12 +346,12 @@ jobs:
312346
*) echo >&2 "bug: bad arch selected"; exit 1;;
313347
esac
314348
./download-kernel.sh ${{ matrix.kernel }}
315-
- name: Run Full Distro tests in QEMU
349+
- name: Run tests in QEMU
316350
run: |
317351
cd test/distro-qemu
318352
case "${{ matrix.target_arch }}" in
319353
amd64) export QEMU_ARCH=x86_64;;
320354
arm64) export QEMU_ARCH=aarch64;;
321355
*) echo >&2 "bug: bad arch selected"; exit 1;;
322356
esac
323-
./build-and-run.sh ${{ matrix.kernel }}
357+
./run-qemu.sh ${{ matrix.kernel }} initramfs-${{ matrix.target_arch }}.gz

test/distro-qemu/build-and-run.sh

Lines changed: 9 additions & 247 deletions
Original file line numberDiff line numberDiff line change
@@ -1,256 +1,18 @@
11
#!/bin/bash
22
set -ex
33

4-
# Configuration
4+
# Convenience wrapper that builds the initramfs and runs QEMU in one step.
5+
# In CI, these are split into separate jobs for efficiency.
6+
#
7+
# Usage: QEMU_ARCH=x86_64 ./build-and-run.sh <kernel-version>
8+
59
KERNEL_VERSION="${1:-5.10.217}"
6-
QEMU_ARCH="${QEMU_ARCH:-x86_64}"
7-
ROOTFS_DIR=$(mktemp -d /tmp/distro-qemu-rootfs.XXXXXX)
8-
OUTPUT_DIR=$(mktemp -d /tmp/distro-qemu-output.XXXXXX)
9-
KERN_DIR="${KERN_DIR:-ci-kernels}"
10-
PARCAGPU_DIR="${PARCAGPU_DIR:-parcagpu-lib}"
10+
INITRAMFS=$(mktemp /tmp/distro-qemu-initramfs.XXXXXX.gz)
1111

1212
cleanup() {
13-
rm -rf "$ROOTFS_DIR" "$OUTPUT_DIR"
13+
rm -f "$INITRAMFS"
1414
}
1515
trap cleanup EXIT
1616

17-
timer_start() { TIMER_START=$(date +%s); }
18-
timer_end() { echo "::> $1 took $(( $(date +%s) - TIMER_START ))s"; }
19-
20-
# Download parcagpu library
21-
timer_start
22-
PARCAGPU_DIR="${PARCAGPU_DIR}" ./download-parcagpu.sh
23-
timer_end "download parcagpu"
24-
25-
# Determine architecture
26-
case "$QEMU_ARCH" in
27-
x86_64) GOARCH="amd64";;
28-
aarch64) GOARCH="arm64";;
29-
*) echo "Unsupported QEMU_ARCH: $QEMU_ARCH"; exit 1;;
30-
esac
31-
32-
# Build test binaries (must be dynamic for dlopen to work)
33-
timer_start
34-
echo "Building test binaries for $GOARCH..."
35-
REPO_ROOT="$(cd ../.. && pwd)"
36-
TEST_PKGS="./interpreter/rtld ./support/usdt/test ./test/cudaverify"
37-
(
38-
cd "${REPO_ROOT}"
39-
if [ "$GOARCH" = "arm64" ] && [ "$(uname -m)" = "x86_64" ]; then
40-
CGO_ENABLED=1 GOARCH=${GOARCH} CC=aarch64-linux-gnu-gcc \
41-
go test -c -o "${OUTPUT_DIR}/" ${TEST_PKGS}
42-
else
43-
CGO_ENABLED=1 GOARCH=${GOARCH} \
44-
go test -c -o "${OUTPUT_DIR}/" ${TEST_PKGS}
45-
fi
46-
)
47-
timer_end "go test -c (build test binaries)"
48-
49-
# Create minimal rootfs (busybox + shared libs only, no debootstrap)
50-
timer_start
51-
echo "Creating minimal rootfs..."
52-
mkdir -p "$ROOTFS_DIR"/{bin,proc,sys,dev,tmp}
53-
54-
# Install busybox for shell and basic utilities
55-
BUSYBOX=$(command -v busybox 2>/dev/null || true)
56-
if [ -z "$BUSYBOX" ]; then
57-
echo "ERROR: busybox not found. Install busybox-static."
58-
exit 1
59-
fi
60-
cp "$BUSYBOX" "$ROOTFS_DIR/bin/busybox"
61-
chmod +x "$ROOTFS_DIR/bin/busybox"
62-
for cmd in sh mount umount dmesg poweroff halt reboot hostname uname find head tail sleep cat grep; do
63-
ln -sf busybox "$ROOTFS_DIR/bin/$cmd"
64-
done
65-
66-
# Copy shared libraries needed by test binaries
67-
copy_lib_deps() {
68-
local binary="$1"
69-
# Copy ELF interpreter
70-
local interp
71-
interp=$(readelf -l "$binary" 2>/dev/null | grep -oP 'Requesting program interpreter: \K[^\]]+' || true)
72-
if [ -n "$interp" ] && [ -f "$interp" ]; then
73-
install -Dm755 "$interp" "$ROOTFS_DIR$interp"
74-
fi
75-
# Copy all shared library dependencies
76-
ldd "$binary" 2>/dev/null | while read -r line; do
77-
# Match lines like: libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x...)
78-
lib=$(echo "$line" | grep -oP '=> \K/\S+' || true)
79-
if [ -n "$lib" ] && [ -f "$lib" ] && [ ! -f "$ROOTFS_DIR$lib" ]; then
80-
real=$(readlink -f "$lib")
81-
install -Dm755 "$real" "$ROOTFS_DIR$real"
82-
# Preserve symlink if the path differs from the real file
83-
if [ "$real" != "$lib" ]; then
84-
mkdir -p "$ROOTFS_DIR$(dirname "$lib")"
85-
ln -sf "$real" "$ROOTFS_DIR$lib"
86-
fi
87-
fi
88-
done
89-
}
90-
91-
for binary in "${OUTPUT_DIR}"/*.test; do
92-
copy_lib_deps "$binary"
93-
done
94-
95-
# Ensure libm.so is available for rtld test (it dlopens libm at runtime,
96-
# so it won't appear in ldd output)
97-
LIBM=$(find /lib* /usr/lib* -name 'libm.so.6' -type f 2>/dev/null | head -1)
98-
if [ -n "$LIBM" ] && [ ! -f "$ROOTFS_DIR$LIBM" ]; then
99-
real=$(readlink -f "$LIBM")
100-
install -Dm755 "$real" "$ROOTFS_DIR$real"
101-
if [ "$real" != "$LIBM" ]; then
102-
mkdir -p "$ROOTFS_DIR$(dirname "$LIBM")"
103-
ln -sf "$real" "$ROOTFS_DIR$LIBM"
104-
fi
105-
fi
106-
107-
# Copy test binaries and parcagpu .so into rootfs
108-
cp "${OUTPUT_DIR}"/*.test "$ROOTFS_DIR/"
109-
cp "${PARCAGPU_DIR}/libparcagpucupti.so" "$ROOTFS_DIR/"
110-
111-
# Show what we have for debugging
112-
echo "Test binary dependencies:"
113-
ldd "${OUTPUT_DIR}/rtld.test" || true
114-
echo ""
115-
echo "Rootfs contents:"
116-
find "$ROOTFS_DIR" -type f -o -type l | sort
117-
echo ""
118-
119-
# Create init script
120-
cat << 'INIT_EOF' > "$ROOTFS_DIR/init"
121-
#!/bin/sh
122-
export PATH=/bin
123-
124-
echo "===== Test Environment ====="
125-
echo "Kernel: $(uname -r)"
126-
127-
# Mount required filesystems
128-
mount -t proc proc /proc 2>/dev/null || true
129-
mount -t sysfs sys /sys 2>/dev/null || true
130-
mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null || true
131-
132-
# Run the tests
133-
echo ""
134-
RESULT=0
135-
for test_bin in /rtld.test /test.test "/cudaverify.test -so-path=/libparcagpucupti.so"; do
136-
name=$(echo "$test_bin" | cut -d' ' -f1)
137-
T0=$(cat /proc/uptime | cut -d' ' -f1)
138-
$test_bin -test.v
139-
rc=$?
140-
T1=$(cat /proc/uptime | cut -d' ' -f1)
141-
echo "::> $name took ${T0}s-${T1}s (uptime)"
142-
if [ $rc -ne 0 ]; then
143-
RESULT=$rc
144-
break
145-
fi
146-
done
147-
148-
if [ $RESULT -eq 0 ]; then
149-
echo ""
150-
echo "===== TEST PASSED ====="
151-
elif [ $RESULT -eq 137 ] || [ $RESULT -eq 124 ]; then
152-
echo ""
153-
echo "===== TEST TIMED OUT ====="
154-
else
155-
echo ""
156-
echo "===== BPF dmesg ====="
157-
dmesg | tail -60
158-
echo "===== TEST FAILED (exit code: $RESULT) ====="
159-
fi
160-
161-
# Shutdown
162-
sleep 1
163-
echo o > /proc/sysrq-trigger 2>/dev/null
164-
sleep 1
165-
poweroff -f 2>/dev/null || halt -f
166-
INIT_EOF
167-
chmod +x "$ROOTFS_DIR/init"
168-
169-
timer_end "create rootfs"
170-
171-
# Create initramfs
172-
timer_start
173-
echo "Creating initramfs..."
174-
(cd "$ROOTFS_DIR" && find . | cpio -o -H newc | gzip > "$OUTPUT_DIR/initramfs.gz")
175-
echo "Rootfs created: $OUTPUT_DIR/initramfs.gz ($(du -h $OUTPUT_DIR/initramfs.gz | cut -f1))"
176-
timer_end "create initramfs"
177-
178-
# Check if kernel exists
179-
if [[ ! -f "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" ]]; then
180-
echo ""
181-
echo "ERROR: Kernel ${KERNEL_VERSION} not found at ${KERN_DIR}/${KERNEL_VERSION}/vmlinuz"
182-
echo ""
183-
echo "To download kernel images:"
184-
echo " mkdir -p ci-kernels"
185-
echo " docker pull ghcr.io/cilium/ci-kernels:${KERNEL_VERSION}"
186-
echo " docker create --name kernel-extract ghcr.io/cilium/ci-kernels:${KERNEL_VERSION}"
187-
echo " docker cp kernel-extract:/boot ci-kernels/${KERNEL_VERSION}"
188-
echo " docker rm kernel-extract"
189-
echo ""
190-
exit 1
191-
fi
192-
193-
# Use sudo if /dev/kvm isn't accessible by the current user
194-
sudo=""
195-
if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then
196-
sudo="sudo"
197-
fi
198-
199-
# Determine additional QEMU args based on architecture
200-
additionalQemuArgs=""
201-
if [ -e /dev/kvm ] && [ "$QEMU_ARCH" = "$(uname -m)" ]; then
202-
additionalQemuArgs="-enable-kvm"
203-
fi
204-
205-
case "$QEMU_ARCH" in
206-
x86_64)
207-
CONSOLE_ARG="console=ttyS0"
208-
;;
209-
aarch64)
210-
additionalQemuArgs+=" -machine virt -cpu max"
211-
CONSOLE_ARG="console=ttyAMA0"
212-
;;
213-
esac
214-
215-
echo ""
216-
echo "===== Starting QEMU with kernel ${KERNEL_VERSION} on ${QEMU_ARCH} ====="
217-
echo ""
218-
219-
# Run QEMU and capture output
220-
timer_start
221-
QEMU_OUTPUT=$(mktemp)
222-
${sudo} qemu-system-${QEMU_ARCH} ${additionalQemuArgs} \
223-
-nographic \
224-
-monitor none \
225-
-serial mon:stdio \
226-
-m 2G \
227-
-kernel "${KERN_DIR}/${KERNEL_VERSION}/vmlinuz" \
228-
-initrd "$OUTPUT_DIR/initramfs.gz" \
229-
-append "${CONSOLE_ARG} init=/init quiet loglevel=3" \
230-
-no-reboot \
231-
-display none \
232-
| tee "$QEMU_OUTPUT"
233-
timer_end "QEMU execution"
234-
235-
# Parse output for test result
236-
if grep -q "===== TEST PASSED =====" "$QEMU_OUTPUT"; then
237-
rm -f "$QEMU_OUTPUT"
238-
echo ""
239-
echo "Test completed successfully"
240-
exit 0
241-
elif grep -q "===== TEST FAILED" "$QEMU_OUTPUT"; then
242-
rm -f "$QEMU_OUTPUT"
243-
echo ""
244-
echo "Test failed"
245-
exit 1
246-
elif grep -q "===== TEST TIMED OUT =====" "$QEMU_OUTPUT"; then
247-
rm -f "$QEMU_OUTPUT"
248-
echo ""
249-
echo "Test timed out"
250-
exit 124
251-
else
252-
rm -f "$QEMU_OUTPUT"
253-
echo ""
254-
echo "Could not determine test result (QEMU may have crashed)"
255-
exit 2
256-
fi
17+
./build-initramfs.sh "$INITRAMFS"
18+
./run-qemu.sh "$KERNEL_VERSION" "$INITRAMFS"

0 commit comments

Comments
 (0)