benchmarks.yml

name: Benchmarks (perf + density)

on:
  workflow_dispatch:
  pull_request:
    branches: [main]
  push:
    branches: [main]
    paths:
      - 'host/**'
      - '.github/workflows/benchmarks.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Measures pyhl performance and density on both Linux and Windows.
#
# Performance: times `pyhl run` across N iterations for hello-world and
# pandas workloads. Reports median, avg, min, max.
#
# Density: launches concurrent VMs running a long computation and
# measures per-VM private memory.
#
# The python-agent-driver image is built on Linux and shared via
# artifact upload (Windows can't build kraft images natively).

jobs:
  # Build the python-agent-driver image once on Linux, upload for both
  # platform jobs to consume.
  build-image:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: read
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: '1.25.1'
          cache: false

      - name: Install just
        uses: extractions/setup-just@v2

      - name: Cache kraft-hyperlight
        id: kraft-cache
        uses: actions/cache@v4
        with:
          path: /usr/local/bin/kraft-hyperlight
          key: kraft-hyperlight-linux-${{ hashFiles('.github/workflows/benchmarks.yml') }}

      - name: Build kraft-hyperlight
        if: steps.kraft-cache.outputs.cache-hit != 'true'
        run: |
          git clone --branch hyperlight-platform --depth 1 \
              https://github.com/danbugs/kraftkit.git /tmp/kraftkit
          cd /tmp/kraftkit && go build -o kraft-hyperlight ./cmd/kraft
          sudo mv kraft-hyperlight /usr/local/bin/

      - name: Build local-python-base images
        env:
          DOCKER_BUILDKIT: '0'
        run: |
          docker build --target base -t local-python-base-dev:latest \
              -f runtimes/python.Dockerfile runtimes/
          docker build -t local-python-base:latest \
              -f runtimes/python.Dockerfile runtimes/

      - name: Build rootfs + kernel
        working-directory: examples/python-agent-driver
        env:
          DOCKER_BUILDKIT: '0'
        run: |
          just rootfs
          kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 || true
          if [ ! -d ".unikraft/unikraft" ] || [ -z "$(ls -A .unikraft/unikraft 2>/dev/null)" ]; then
            echo "::warning::kraft didn't clone unikraft sources; cloning manually"
            UK_SOURCE=$(awk '/^unikraft:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            UK_BRANCH=$(awk '/^unikraft:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            ELF_SOURCE=$(awk '/app-elfloader:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            ELF_BRANCH=$(awk '/app-elfloader:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            mkdir -p .unikraft/apps .unikraft/libs
            rm -rf .unikraft/unikraft .unikraft/apps/elfloader .unikraft/libs/libelf .unikraft/build
            git clone --branch "$UK_BRANCH" --depth 1 "$UK_SOURCE" .unikraft/unikraft
            git clone --branch "$ELF_BRANCH" --depth 1 "$ELF_SOURCE" .unikraft/apps/elfloader
            git clone --branch staging --depth 1 https://github.com/unikraft/lib-libelf.git .unikraft/libs/libelf
            kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64
          fi

      - name: Package image
        working-directory: examples/python-agent-driver
        run: |
          mkdir -p /tmp/image
          cp .unikraft/build/*_hyperlight-x86_64 /tmp/image/kernel
          cpio_file=$(ls *-initrd.cpio 2>/dev/null | head -1)
          if [ -z "$cpio_file" ]; then
            cpio_file=$(ls initrd.cpio 2>/dev/null | head -1)
          fi
          cp "$cpio_file" /tmp/image/initrd.cpio
          ls -la /tmp/image/

      - name: Upload image artifact
        uses: actions/upload-artifact@v4
        with:
          name: bench-image
          path: /tmp/image/
          retention-days: 1
          if-no-files-found: error

  # ------------------------------------------------------------------
  # Performance + density on Linux
  # ------------------------------------------------------------------
  bench-linux:
    runs-on: ubuntu-latest
    needs: build-image
    permissions:
      contents: write
      packages: read
      pull-requests: write
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: host -> target

      - name: Enable KVM permissions
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm || true

      - name: Check KVM availability
        id: kvm_check
        run: |
          if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then
            echo "available=true" >> $GITHUB_OUTPUT
          else
            echo "available=false" >> $GITHUB_OUTPUT
            echo "::warning::/dev/kvm not available; benchmarks skipped"
          fi

      - name: Install pyhl
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          cd host
          cargo build --release --bin pyhl
          sudo cp target/release/pyhl /usr/local/bin/

      - name: Download prebuilt image
        if: steps.kvm_check.outputs.available == 'true'
        uses: actions/download-artifact@v4
        with:
          name: bench-image
          path: bench-image

      - name: Lay out pyhl source dir
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          mkdir -p src-dir/.unikraft/build
          mv bench-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64
          mv bench-image/initrd.cpio src-dir/pyhl-initrd.cpio

      - name: pyhl setup
        if: steps.kvm_check.outputs.available == 'true'
        run: pyhl setup --from src-dir --force

      - name: "Smoke: verify pyhl run works"
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          echo "--- snapshot and system info ---"
          ls -lh .pyhl/
          free -h
          echo "--- smoke test ---"
          pyhl run -c "print('smoke ok')" || { echo "FAIL: pyhl run exited $?"; exit 1; }

      - name: "Perf: hello world (15 runs)"
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          echo "=== Hello World — print(42) ==="
          times=()
          for i in $(seq 1 15); do
            ms=$( { /usr/bin/time -f "%e" pyhl run -c "print(42)" > /dev/null; } 2>&1 )
            ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1)
            times+=($ms_int)
            echo "  run $i: ${ms_int}ms"
          done
          IFS=$'\n' sorted=($(sort -n <<<"${times[*]}")); unset IFS
          n=${#sorted[@]}
          mid=$((n / 2))
          median=${sorted[$mid]}
          sum=0; for t in "${times[@]}"; do sum=$((sum + t)); done
          avg=$((sum / n))
          min=${sorted[0]}
          max=${sorted[$((n - 1))]}
          echo ""
          echo "| Metric | Value |"
          echo "|--------|-------|"
          echo "| Median | ${median}ms |"
          echo "| Avg    | ${avg}ms |"
          echo "| Min    | ${min}ms |"
          echo "| Max    | ${max}ms |"
          echo "| Runs   | $n |"
          echo ""
          echo "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
          echo "{\"name\":\"hello_world (median)\",\"unit\":\"ms\",\"value\":$median}" >> /tmp/bench-results.jsonl

      - name: "Perf: pandas (10 runs)"
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          echo "=== Pandas — DataFrame.describe() ==="
          cat > /tmp/pandas_bench.py << 'PYEOF'
          import pandas as pd, numpy as np
          df = pd.DataFrame(np.random.randn(1000, 4), columns=list('ABCD'))
          print(df.describe())
          PYEOF
          pyhl run /tmp/pandas_bench.py > /dev/null || { echo "FAIL: pandas smoke check failed"; exit 1; }
          times=()
          for i in $(seq 1 10); do
            ms=$( { /usr/bin/time -f "%e" pyhl run /tmp/pandas_bench.py > /dev/null; } 2>&1 )
            ms_int=$(echo "$ms * 1000" | bc | cut -d. -f1)
            times+=($ms_int)
            echo "  run $i: ${ms_int}ms"
          done
          IFS=$'\n' sorted=($(sort -n <<<"${times[*]}")); unset IFS
          n=${#sorted[@]}
          mid=$((n / 2))
          median=${sorted[$mid]}
          sum=0; for t in "${times[@]}"; do sum=$((sum + t)); done
          avg=$((sum / n))
          min=${sorted[0]}
          max=${sorted[$((n - 1))]}
          echo ""
          echo "| Metric | Value |"
          echo "|--------|-------|"
          echo "| Median | ${median}ms |"
          echo "| Avg    | ${avg}ms |"
          echo "| Min    | ${min}ms |"
          echo "| Max    | ${max}ms |"
          echo "| Runs   | $n |"
          echo ""
          echo "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
          echo "{\"name\":\"pandas (median)\",\"unit\":\"ms\",\"value\":$median}" >> /tmp/bench-results.jsonl

      - name: "Density: concurrent VMs (5 VMs)"
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          echo "=== Density — 5 concurrent VMs ==="
          N=5
          pids=()
          for i in $(seq 1 $N); do
            pyhl run -c "x = sum(range(500_000_000)); print($i, x)" &
            pids+=($!)
          done
          sleep 8
          total_private=0; count=0
          for pid in "${pids[@]}"; do
            if [ -f /proc/$pid/smaps_rollup ]; then
              private=$(grep "Private_Dirty" /proc/$pid/smaps_rollup | awk '{sum+=$2} END {print sum}')
              if [ -n "$private" ] && [ "$private" -gt 0 ]; then
                echo "  PID $pid: Private_Dirty=${private} kB ($((private / 1024)) MB)"
                total_private=$((total_private + private)); count=$((count + 1))
              fi
            fi
          done
          if [ "$count" -gt 0 ]; then
            per_vm=$((total_private / count / 1024))
            echo ""
            echo "| Metric | Value |"
            echo "|--------|-------|"
            echo "| VMs measured | $count |"
            echo "| Per-VM Private_Dirty | ${per_vm} MB |"
            echo "| Total Private_Dirty | $((total_private / 1024)) MB |"
            echo ""
            echo "::notice::density: ${count} VMs, per_vm=${per_vm}MB private_dirty"
            echo "{\"name\":\"density (per VM)\",\"unit\":\"MB\",\"value\":$per_vm}" >> /tmp/bench-results.jsonl
          else
            echo "::warning::no VMs were alive long enough to measure"
          fi
          wait

      - name: "Snapshot size"
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          snap=".pyhl/snapshot"
          apparent=$(($(find "$snap" -type f -exec stat -c '%s' {} + | paste -sd+ | bc) / 1024 / 1024))
          disk=$(($(find "$snap" -type f -exec stat -c '%b' {} + | paste -sd+ | bc) * 512 / 1024 / 1024))
          echo "| Metric | Value |"
          echo "|--------|-------|"
          echo "| Apparent size | ${apparent} MiB |"
          echo "| Disk usage | ${disk} MiB |"
          echo ""
          echo "::notice::snapshot: apparent=${apparent}MiB disk=${disk}MiB"
          echo "{\"name\":\"snapshot (disk)\",\"unit\":\"MiB\",\"value\":$disk}" >> /tmp/bench-results.jsonl

      - name: Collect benchmark results
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          if [ -f /tmp/bench-results.jsonl ]; then
            jq -s '.' /tmp/bench-results.jsonl > /tmp/bench-results.json
          else
            echo "[]" > /tmp/bench-results.json
          fi
          cat /tmp/bench-results.json

      - name: Store benchmark results
        if: steps.kvm_check.outputs.available == 'true'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Linux Benchmarks
          tool: customSmallerIsBetter
          output-file-path: /tmp/bench-results.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: dev/bench/linux
          auto-push: ${{ github.event_name == 'push' }}
          comment-always: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
          alert-threshold: '130%'

  # ------------------------------------------------------------------
  # Performance + density on Windows
  # ------------------------------------------------------------------
  bench-windows:
    runs-on: windows-latest
    needs: [build-image, bench-linux]
    permissions:
      contents: write
      packages: read
      pull-requests: write
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: host -> target

      - name: Ensure surrogate build consistency
        shell: pwsh
        run: |
          $hlsExe = "host\target\release\build\hyperlight-host-*\out\..\..\hls\x86_64-pc-windows-msvc\release\hyperlight_surrogate.exe"
          if (-not (Resolve-Path $hlsExe -ErrorAction SilentlyContinue)) {
            Write-Host "Surrogate missing — clearing hyperlight-host fingerprints to force rebuild"
            Get-ChildItem "host\target\release\.fingerprint" -Filter "hyperlight-host-*" -Directory -ErrorAction SilentlyContinue |
              Remove-Item -Recurse -Force
          }

      - name: Install pyhl
        shell: pwsh
        run: |
          cd host
          cargo build --release --bin pyhl
          Copy-Item target\release\pyhl.exe $env:USERPROFILE\.cargo\bin\ -Force

      - name: Download prebuilt image
        uses: actions/download-artifact@v4
        with:
          name: bench-image
          path: bench-image

      - name: Lay out pyhl source dir
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Force -Path src-dir/.unikraft/build | Out-Null
          Move-Item bench-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64
          Move-Item bench-image/initrd.cpio src-dir/pyhl-initrd.cpio

      - name: pyhl setup
        shell: pwsh
        run: pyhl setup --from src-dir --force

      - name: "Perf: hello world (15 runs)"
        shell: pwsh
        run: |
          Write-Host "=== Hello World — print(42) ==="
          $times = @()
          for ($i = 1; $i -le 15; $i++) {
            $elapsed = (Measure-Command {
              $out = pyhl run -c "print(42)"
            }).TotalMilliseconds
            $ms = [math]::Round($elapsed)
            $times += $ms
            Write-Host "  run ${i}: ${ms}ms"
          }
          $sorted = $times | Sort-Object
          $n = $sorted.Count
          $median = $sorted[[math]::Floor($n / 2)]
          $avg = [math]::Round(($times | Measure-Object -Average).Average)
          $min = $sorted[0]
          $max = $sorted[$n - 1]
          Write-Host ""
          Write-Host "| Metric | Value |"
          Write-Host "|--------|-------|"
          Write-Host "| Median | ${median}ms |"
          Write-Host "| Avg    | ${avg}ms |"
          Write-Host "| Min    | ${min}ms |"
          Write-Host "| Max    | ${max}ms |"
          Write-Host "| Runs   | $n |"
          Write-Host ""
          Write-Host "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
          @{name="hello_world (median)"; unit="ms"; value=$median} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii

      - name: "Perf: pandas (10 runs)"
        shell: pwsh
        run: |
          Write-Host "=== Pandas — DataFrame.describe() ==="
          @"
          import pandas as pd, numpy as np
          df = pd.DataFrame(np.random.randn(1000, 4), columns=list('ABCD'))
          print(df.describe())
          "@ | Out-File -Encoding ascii pandas_bench.py
          $times = @()
          for ($i = 1; $i -le 10; $i++) {
            $elapsed = (Measure-Command {
              $out = pyhl run pandas_bench.py
            }).TotalMilliseconds
            $ms = [math]::Round($elapsed)
            $times += $ms
            Write-Host "  run ${i}: ${ms}ms"
          }
          $sorted = $times | Sort-Object
          $n = $sorted.Count
          $median = $sorted[[math]::Floor($n / 2)]
          $avg = [math]::Round(($times | Measure-Object -Average).Average)
          $min = $sorted[0]
          $max = $sorted[$n - 1]
          Write-Host ""
          Write-Host "| Metric | Value |"
          Write-Host "|--------|-------|"
          Write-Host "| Median | ${median}ms |"
          Write-Host "| Avg    | ${avg}ms |"
          Write-Host "| Min    | ${min}ms |"
          Write-Host "| Max    | ${max}ms |"
          Write-Host "| Runs   | $n |"
          Write-Host ""
          Write-Host "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
          @{name="pandas (median)"; unit="ms"; value=$median} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii

      - name: "Density: concurrent VMs (5 VMs)"
        shell: pwsh
        run: |
          Write-Host "=== Density — 5 concurrent VMs ==="
          $procs = @()
          for ($i = 1; $i -le 5; $i++) {
            $p = Start-Process -FilePath "pyhl" `
              -ArgumentList "run -c `"x=sum(range(500_000_000)); print($i, x)`"" `
              -PassThru -NoNewWindow -RedirectStandardOutput "NUL"
            $procs += $p
          }
          Start-Sleep -Seconds 10
          $totalPrivate = 0; $count = 0
          foreach ($proc in $procs) {
            $proc.Refresh()
            if (-not $proc.HasExited) {
              $privMB = [math]::Round($proc.PrivateMemorySize64 / 1MB, 0)
              $wsMB = [math]::Round($proc.WorkingSet64 / 1MB, 0)
              Write-Host "  PID $($proc.Id): Private=${privMB}MB WS=${wsMB}MB"
              $totalPrivate += $privMB; $count++
            }
          }
          if ($count -gt 0) {
            $perVM = [math]::Round($totalPrivate / $count)
            Write-Host ""
            Write-Host "| Metric | Value |"
            Write-Host "|--------|-------|"
            Write-Host "| VMs measured | $count |"
            Write-Host "| Per-VM Private | ${perVM} MB |"
            Write-Host "| Total Private | ${totalPrivate} MB |"
            Write-Host ""
            Write-Host "::notice::density: ${count} VMs, per_vm=${perVM}MB private"
            @{name="density (per VM)"; unit="MB"; value=$perVM} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii
          } else {
            Write-Host "::warning::no VMs were alive long enough to measure"
          }
          foreach ($proc in $procs) {
            $proc.WaitForExit(60000) | Out-Null
          }

      - name: "Snapshot size"
        shell: pwsh
        run: |
          Add-Type -TypeDefinition @"
          using System;
          using System.Runtime.InteropServices;
          public class SparseFileHelper {
              [DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)]
              public static extern uint GetCompressedFileSizeW(string lpFileName, out uint lpFileSizeHigh);
          }
          "@
          function Get-CompressedSize($path) {
              $high = [uint32]0
              $low = [SparseFileHelper]::GetCompressedFileSizeW($path, [ref]$high)
              if ($low -eq 0xFFFFFFFF) {
                  $err = [System.Runtime.InteropServices.Marshal]::GetLastWin32Error()
                  if ($err -ne 0) { return (Get-Item $path).Length }
              }
              return ([uint64]$high -shl 32) -bor [uint64]$low
          }
          $snap = ".pyhl\snapshot"
          $files = Get-ChildItem -Path $snap -Recurse -File
          $apparentMiB = [math]::Round(($files | Measure-Object -Property Length -Sum).Sum / 1MB)
          $diskBytes = [uint64]0
          foreach ($f in $files) { $diskBytes += Get-CompressedSize $f.FullName }
          $diskMiB = [math]::Round($diskBytes / 1MB)
          Write-Host "| Metric | Value |"
          Write-Host "|--------|-------|"
          Write-Host "| Apparent size | ${apparentMiB} MiB |"
          Write-Host "| Disk usage | ${diskMiB} MiB |"
          Write-Host ""
          Write-Host "::notice::snapshot: apparent=${apparentMiB}MiB disk=${diskMiB}MiB"
          @{name="snapshot (disk)"; unit="MiB"; value=$diskMiB} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii

      - name: Collect benchmark results
        shell: pwsh
        run: |
          if (Test-Path bench-results.jsonl) {
            $lines = Get-Content bench-results.jsonl
            $json = "[" + ($lines -join ",") + "]"
            $json | Out-File bench-results.json -Encoding ascii
          } else {
            "[]" | Out-File bench-results.json -Encoding ascii
          }
          Get-Content bench-results.json

      - name: Store benchmark results
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Windows Benchmarks
          tool: customSmallerIsBetter
          output-file-path: bench-results.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: dev/bench/windows
          auto-push: ${{ github.event_name == 'push' }}
          comment-always: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
          alert-threshold: '130%'

  benchmarks-passed:
    if: always()
    needs: [build-image, bench-linux, bench-windows]
    runs-on: ubuntu-latest
    permissions: {}
    steps:
      - run: |
          declare -A results=(
            [build-image]="${{ needs.build-image.result }}"
            [bench-linux]="${{ needs.bench-linux.result }}"
            [bench-windows]="${{ needs.bench-windows.result }}"
          )
          failed=0
          for job in "${!results[@]}"; do
            r="${results[$job]}"
            if [[ "$r" != "success" && "$r" != "skipped" ]]; then
              echo "FAIL: $job = $r"
              failed=1
            fi
          done
          if [[ "$failed" -eq 1 ]]; then
            exit 1
          fi
          echo "All checks passed"