diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..1c52f9d --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,446 @@ +name: Benchmarks (perf + density) + +on: + workflow_dispatch: + pull_request: + branches: [main] + paths: + - 'host/**' + - '.github/workflows/benchmarks.yml' + push: + branches: [main] + paths: + - 'host/**' + - '.github/workflows/benchmarks.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Measures pyhl performance and density on both Linux and Windows. +# +# Performance: times `pyhl run` across N iterations for hello-world and +# pandas workloads. Reports median, avg, min, max. +# +# Density: launches concurrent VMs running a long computation and +# measures per-VM private memory. +# +# The python-agent-driver image is built on Linux and shared via +# artifact upload (Windows can't build kraft images natively). + +jobs: + # Build the python-agent-driver image once on Linux, upload for both + # platform jobs to consume. + build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install Go (for kraft-hyperlight) + run: | + curl -sL https://go.dev/dl/go1.25.1.linux-amd64.tar.gz | sudo tar -C /usr/local -xz + echo "/usr/local/go/bin" >> $GITHUB_PATH + + - name: Install just + uses: extractions/setup-just@v2 + + - name: Build kraft-hyperlight + run: | + git clone --branch hyperlight-platform --depth 1 \ + https://github.com/danbugs/kraftkit.git /tmp/kraftkit + cd /tmp/kraftkit && go build -o kraft-hyperlight ./cmd/kraft + sudo mv kraft-hyperlight /usr/local/bin/ + + - name: Build local-python-base images + env: + DOCKER_BUILDKIT: '0' + run: | + docker build --target base -t local-python-base-dev:latest \ + -f runtimes/python.Dockerfile runtimes/ + docker build -t local-python-base:latest \ + -f runtimes/python.Dockerfile runtimes/ + + - name: Build rootfs + kernel + working-directory: examples/python-agent-driver + env: + DOCKER_BUILDKIT: '0' + run: | + just rootfs + kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 || true + if [ ! -d ".unikraft/unikraft" ] || [ -z "$(ls -A .unikraft/unikraft 2>/dev/null)" ]; then + echo "::warning::kraft didn't clone unikraft sources; cloning manually" + UK_BRANCH=$(grep -E '^ version:' kraft.yaml | head -1 | awk '{print $2}') + mkdir -p .unikraft/apps .unikraft/libs + rm -rf .unikraft/unikraft .unikraft/apps/elfloader .unikraft/libs/libelf .unikraft/build + git clone --branch $UK_BRANCH --depth 1 https://github.com/unikraft/unikraft.git .unikraft/unikraft + git clone --branch $UK_BRANCH --depth 1 https://github.com/unikraft/app-elfloader.git .unikraft/apps/elfloader + git clone --branch staging --depth 1 https://github.com/unikraft/lib-libelf.git .unikraft/libs/libelf + kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 + fi + + - name: Package image + working-directory: examples/python-agent-driver + run: | + mkdir -p /tmp/image + cp .unikraft/build/*_hyperlight-x86_64 /tmp/image/kernel + cpio_file=$(ls *-initrd.cpio 2>/dev/null | head -1) + if [ -z "$cpio_file" ]; then + cpio_file=$(ls initrd.cpio 2>/dev/null | head -1) + fi + cp "$cpio_file" /tmp/image/initrd.cpio + ls -la /tmp/image/ + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: bench-image + path: /tmp/image/ + retention-days: 1 + if-no-files-found: error + + # ------------------------------------------------------------------ + # Performance + density on Linux + # ------------------------------------------------------------------ + bench-linux: + runs-on: ubuntu-latest + needs: build-image + permissions: + contents: read + packages: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Enable KVM permissions + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm || true + + - name: Check KVM availability + id: kvm_check + run: | + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "::warning::/dev/kvm not available; benchmarks skipped" + fi + + - name: Install pyhl + if: steps.kvm_check.outputs.available == 'true' + run: | + cd host + cargo build --release --bin pyhl + sudo cp target/release/pyhl /usr/local/bin/ + + - name: Download prebuilt image + if: steps.kvm_check.outputs.available == 'true' + uses: actions/download-artifact@v4 + with: + name: bench-image + path: bench-image + + - name: Lay out pyhl source dir + if: steps.kvm_check.outputs.available == 'true' + run: | + mkdir -p src-dir/.unikraft/build + mv bench-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64 + mv bench-image/initrd.cpio src-dir/pyhl-initrd.cpio + + - name: pyhl setup + if: steps.kvm_check.outputs.available == 'true' + run: pyhl setup --from src-dir --force + + - name: "Perf: hello world (15 runs)" + if: steps.kvm_check.outputs.available == 'true' + run: | + echo "=== Hello World — print(42) ===" + times=() + for i in $(seq 1 15); do + ms=$( { /usr/bin/time -f "%e" pyhl run -c "print(42)" > /dev/null; } 2>&1 ) + ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1) + times+=($ms_int) + echo " run $i: ${ms_int}ms" + done + IFS=$'\n' sorted=($(sort -n <<<"${times[*]}")); unset IFS + n=${#sorted[@]} + mid=$((n / 2)) + median=${sorted[$mid]} + sum=0; for t in "${times[@]}"; do sum=$((sum + t)); done + avg=$((sum / n)) + min=${sorted[0]} + max=${sorted[$((n - 1))]} + echo "" + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Median | ${median}ms |" + echo "| Avg | ${avg}ms |" + echo "| Min | ${min}ms |" + echo "| Max | ${max}ms |" + echo "| Runs | $n |" + echo "" + echo "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms" + + - name: "Perf: pandas (10 runs)" + if: steps.kvm_check.outputs.available == 'true' + run: | + echo "=== Pandas — DataFrame.describe() ===" + cat > /tmp/pandas_bench.py << 'PYEOF' + import pandas as pd, numpy as np + df = pd.DataFrame(np.random.randn(1000, 4), columns=list('ABCD')) + print(df.describe()) + PYEOF + times=() + for i in $(seq 1 10); do + ms=$( { /usr/bin/time -f "%e" pyhl run /tmp/pandas_bench.py; } 2>&1 | tail -1 ) + ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1) + times+=($ms_int) + echo " run $i: ${ms_int}ms" + done + IFS=$'\n' sorted=($(sort -n <<<"${times[*]}")); unset IFS + n=${#sorted[@]} + mid=$((n / 2)) + median=${sorted[$mid]} + sum=0; for t in "${times[@]}"; do sum=$((sum + t)); done + avg=$((sum / n)) + min=${sorted[0]} + max=${sorted[$((n - 1))]} + echo "" + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Median | ${median}ms |" + echo "| Avg | ${avg}ms |" + echo "| Min | ${min}ms |" + echo "| Max | ${max}ms |" + echo "| Runs | $n |" + echo "" + echo "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms" + + - name: "Density: concurrent VMs (5 VMs)" + if: steps.kvm_check.outputs.available == 'true' + run: | + echo "=== Density — 5 concurrent VMs ===" + N=5 + pids=() + for i in $(seq 1 $N); do + pyhl run -c "x = sum(range(500_000_000)); print($i, x)" & + pids+=($!) + done + sleep 8 + total_private=0; count=0 + for pid in "${pids[@]}"; do + if [ -f /proc/$pid/smaps_rollup ]; then + private=$(grep "Private_Dirty" /proc/$pid/smaps_rollup | awk '{sum+=$2} END {print sum}') + if [ -n "$private" ] && [ "$private" -gt 0 ]; then + echo " PID $pid: Private_Dirty=${private} kB ($((private / 1024)) MB)" + total_private=$((total_private + private)); count=$((count + 1)) + fi + fi + done + if [ "$count" -gt 0 ]; then + per_vm=$((total_private / count / 1024)) + echo "" + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| VMs measured | $count |" + echo "| Per-VM Private_Dirty | ${per_vm} MB |" + echo "| Total Private_Dirty | $((total_private / 1024)) MB |" + echo "" + echo "::notice::density: ${count} VMs, per_vm=${per_vm}MB private_dirty" + else + echo "::warning::no VMs were alive long enough to measure" + fi + wait + + - name: "Snapshot size" + if: steps.kvm_check.outputs.available == 'true' + run: | + snap=".pyhl/snapshot.hls" + apparent=$(($(stat -c '%s' "$snap") / 1024 / 1024)) + disk=$(($(stat -c '%b' "$snap") * 512 / 1024 / 1024)) + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Apparent size | ${apparent} MiB |" + echo "| Disk usage | ${disk} MiB |" + echo "" + echo "::notice::snapshot: apparent=${apparent}MiB disk=${disk}MiB" + + # ------------------------------------------------------------------ + # Performance + density on Windows + # ------------------------------------------------------------------ + bench-windows: + runs-on: windows-latest + needs: build-image + permissions: + contents: read + packages: read + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Install pyhl + shell: pwsh + run: | + cargo install --git https://github.com/${{ github.repository }} ` + --branch ${{ github.head_ref || github.ref_name }} ` + hyperlight-unikraft-host ` + --bin pyhl ` + --locked --force + + - name: Download prebuilt image + uses: actions/download-artifact@v4 + with: + name: bench-image + path: bench-image + + - name: Lay out pyhl source dir + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path src-dir/.unikraft/build | Out-Null + Move-Item bench-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64 + Move-Item bench-image/initrd.cpio src-dir/pyhl-initrd.cpio + + - name: pyhl setup + shell: pwsh + run: pyhl setup --from src-dir --force + + - name: "Perf: hello world (15 runs)" + shell: pwsh + run: | + Write-Host "=== Hello World — print(42) ===" + $times = @() + for ($i = 1; $i -le 15; $i++) { + $elapsed = (Measure-Command { + $out = pyhl run -c "print(42)" + }).TotalMilliseconds + $ms = [math]::Round($elapsed) + $times += $ms + Write-Host " run ${i}: ${ms}ms" + } + $sorted = $times | Sort-Object + $n = $sorted.Count + $median = $sorted[[math]::Floor($n / 2)] + $avg = [math]::Round(($times | Measure-Object -Average).Average) + $min = $sorted[0] + $max = $sorted[$n - 1] + Write-Host "" + Write-Host "| Metric | Value |" + Write-Host "|--------|-------|" + Write-Host "| Median | ${median}ms |" + Write-Host "| Avg | ${avg}ms |" + Write-Host "| Min | ${min}ms |" + Write-Host "| Max | ${max}ms |" + Write-Host "| Runs | $n |" + Write-Host "" + Write-Host "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms" + + - name: "Perf: pandas (10 runs)" + shell: pwsh + run: | + Write-Host "=== Pandas — DataFrame.describe() ===" + @" + import pandas as pd, numpy as np + df = pd.DataFrame(np.random.randn(1000, 4), columns=list('ABCD')) + print(df.describe()) + "@ | Out-File -Encoding ascii pandas_bench.py + $times = @() + for ($i = 1; $i -le 10; $i++) { + $elapsed = (Measure-Command { + $out = pyhl run pandas_bench.py + }).TotalMilliseconds + $ms = [math]::Round($elapsed) + $times += $ms + Write-Host " run ${i}: ${ms}ms" + } + $sorted = $times | Sort-Object + $n = $sorted.Count + $median = $sorted[[math]::Floor($n / 2)] + $avg = [math]::Round(($times | Measure-Object -Average).Average) + $min = $sorted[0] + $max = $sorted[$n - 1] + Write-Host "" + Write-Host "| Metric | Value |" + Write-Host "|--------|-------|" + Write-Host "| Median | ${median}ms |" + Write-Host "| Avg | ${avg}ms |" + Write-Host "| Min | ${min}ms |" + Write-Host "| Max | ${max}ms |" + Write-Host "| Runs | $n |" + Write-Host "" + Write-Host "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms" + + - name: "Density: concurrent VMs (5 VMs)" + shell: pwsh + run: | + Write-Host "=== Density — 5 concurrent VMs ===" + $procs = @() + for ($i = 1; $i -le 5; $i++) { + $p = Start-Process -FilePath "pyhl" ` + -ArgumentList "run -c `"x=sum(range(500_000_000)); print($i, x)`"" ` + -PassThru -NoNewWindow -RedirectStandardOutput "NUL" + $procs += $p + } + Start-Sleep -Seconds 10 + $totalPrivate = 0; $count = 0 + foreach ($proc in $procs) { + $proc.Refresh() + if (-not $proc.HasExited) { + $privMB = [math]::Round($proc.PrivateMemorySize64 / 1MB, 0) + $wsMB = [math]::Round($proc.WorkingSet64 / 1MB, 0) + Write-Host " PID $($proc.Id): Private=${privMB}MB WS=${wsMB}MB" + $totalPrivate += $privMB; $count++ + } + } + if ($count -gt 0) { + $perVM = [math]::Round($totalPrivate / $count) + Write-Host "" + Write-Host "| Metric | Value |" + Write-Host "|--------|-------|" + Write-Host "| VMs measured | $count |" + Write-Host "| Per-VM Private | ${perVM} MB |" + Write-Host "| Total Private | ${totalPrivate} MB |" + Write-Host "" + Write-Host "::notice::density: ${count} VMs, per_vm=${perVM}MB private" + } else { + Write-Host "::warning::no VMs were alive long enough to measure" + } + foreach ($proc in $procs) { + $proc.WaitForExit(60000) | Out-Null + } + + - name: "Snapshot size" + shell: pwsh + run: | + $snap = ".pyhl\snapshot.hls" + $f = Get-Item $snap + $apparentMiB = [math]::Round($f.Length / 1MB) + # GetCompressedFileSize returns actual on-disk allocation for sparse files + $wide = $f.FullName + Add-Type -TypeDefinition @" + using System; + using System.Runtime.InteropServices; + public class SparseHelper { + [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] + public static extern uint GetCompressedFileSizeW(string lpFileName, out uint lpFileSizeHigh); + } + "@ -ErrorAction SilentlyContinue + $high = [uint32]0 + $low = [SparseHelper]::GetCompressedFileSizeW($wide, [ref]$high) + if ($low -ne [uint32]::MaxValue) { + $diskBytes = ([uint64]$high -shl 32) -bor [uint64]$low + $diskMiB = [math]::Round($diskBytes / 1MB) + } else { + $diskMiB = $apparentMiB + } + Write-Host "| Metric | Value |" + Write-Host "|--------|-------|" + Write-Host "| Apparent size | ${apparentMiB} MiB |" + Write-Host "| Disk usage | ${diskMiB} MiB |" + Write-Host "" + Write-Host "::notice::snapshot: apparent=${apparentMiB}MiB disk=${diskMiB}MiB"