test-examples.yml

name: Test examples (regression gate)

on:
  workflow_dispatch:
  pull_request:
    branches: [main]
  push:
    branches: [main]
    paths:
      - "examples/**"
      - "runtimes/**"
      - "host/**"
      - ".github/workflows/test-examples.yml"

# Pushing a new commit to the same ref cancels any in-flight run on
# the previous commit — don't waste CI on stale code.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Build-only regression gate — each example is built from scratch
# (rootfs + kernel) in its own job. We do NOT execute the guests
# because GitHub-hosted Linux runners don't expose /dev/kvm; hyperlight
# aborts with "No hypervisor was found" before any guest code runs.
#
# Catches the kinds of regressions that hit us this session:
# missing CONFIG_* guards on renamed kernel branches, NuGet version
# drift, Dockerfile dependency issues. The runtime-test job below
# runs each example under KVM on ubuntu-latest and greps the guest
# output for an expected string, so actual runtime crashes (like the
# cpiovfs-vs-ramfs type-confusion that broke powershell) surface too.

env:
  REGISTRY: ghcr.io
  IMAGE_BASE: ghcr.io/${{ github.repository }}

jobs:
  build-example:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: read
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        example:
          - helloworld-c
          - rust
          - go
          - shell
          - python
          - python-tools
          - nodejs
          - dotnet
          - dotnet-nativeaot
          - hostfs-posix-c
          - hostfs-posix-py
          - multifn-c
          - python-agent
          - python-agent-driver
          - powershell
          - networking-py
          - go-http
          - dotnet-http
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: "1.25.1"
          cache: false

      - name: Install just
        uses: extractions/setup-just@v2

      - name: Cache kraft-hyperlight
        id: kraft-cache
        uses: actions/cache@v4
        with:
          path: /usr/local/bin/kraft-hyperlight
          key: kraft-hyperlight-linux-${{ hashFiles('.github/workflows/test-examples.yml') }}

      - name: Build kraft-hyperlight
        if: steps.kraft-cache.outputs.cache-hit != 'true'
        run: |
          git clone --branch hyperlight-platform --depth 1 \
              https://github.com/danbugs/kraftkit.git /tmp/kraftkit
          cd /tmp/kraftkit && go build -o kraft-hyperlight ./cmd/kraft
          sudo mv kraft-hyperlight /usr/local/bin/

      - name: Build local-python-base images (python-agent-driver only)
        if: matrix.example == 'python-agent-driver'
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          docker build --target base -t local-python-base-dev:latest \
              -f runtimes/python.Dockerfile runtimes/
          docker build -t local-python-base:latest \
              -f runtimes/python.Dockerfile runtimes/

      - name: Build rootfs
        working-directory: examples/${{ matrix.example }}
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          just rootfs

      - name: Build kernel
        working-directory: examples/${{ matrix.example }}
        run: |
          kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 || true
          # Fallback: kraft's source-resolution is flaky — it can partially
          # clone (e.g. unikraft but not libelf) or leave dirs empty. Check
          # for the actual kernel output; if missing, nuke everything and
          # clone + build from scratch.
          if ! ls .unikraft/build/*_hyperlight-x86_64 >/dev/null 2>&1; then
            echo "::warning::kraft build produced no kernel; cloning manually and rebuilding"
            UK_SOURCE=$(awk '/^unikraft:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            UK_BRANCH=$(awk '/^unikraft:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            ELF_SOURCE=$(awk '/app-elfloader:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            ELF_BRANCH=$(awk '/app-elfloader:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            mkdir -p .unikraft/apps .unikraft/libs
            rm -rf .unikraft/unikraft .unikraft/apps/elfloader .unikraft/libs/libelf .unikraft/build
            git clone --branch "$UK_BRANCH" --depth 1 "$UK_SOURCE" .unikraft/unikraft
            git clone --branch "$ELF_BRANCH" --depth 1 "$ELF_SOURCE" .unikraft/apps/elfloader
            git clone --branch staging --depth 1 https://github.com/unikraft/lib-libelf.git .unikraft/libs/libelf
            kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64
          fi

      - name: Verify artifacts produced
        working-directory: examples/${{ matrix.example }}
        run: |
          kernel=$(ls .unikraft/build/*_hyperlight-x86_64 2>/dev/null | head -1)
          cpio=$(ls *-initrd.cpio initrd.cpio 2>/dev/null | head -1)
          if [ -z "$kernel" ] || [ ! -f "$kernel" ]; then
            echo "FAIL: no kernel artifact under .unikraft/build/"
            ls -la .unikraft/build/ 2>/dev/null || echo "  (no .unikraft/build dir)"
            exit 1
          fi
          if [ -z "$cpio" ] || [ ! -f "$cpio" ]; then
            echo "FAIL: no CPIO rootfs produced"
            ls -la *.cpio 2>/dev/null || echo "  (no .cpio files)"
            exit 1
          fi
          echo "PASS: kernel=$(stat -c '%s' $kernel) bytes, cpio=$(stat -c '%s' $cpio) bytes"

  # Runtime test. GitHub-hosted ubuntu-latest runners DO expose /dev/kvm
  # (verified 2026-04-20); the udev snippet below makes the device node
  # world-accessible so Hyperlight can create a VM under the default
  # runner user. A presence check still short-circuits the job with a
  # ::warning:: if a future runner ships without KVM, so it degrades
  # gracefully rather than failing mysteriously.
  #
  # Full matrix — every example actually runs and its expected output is
  # greped from the guest console. This is the real regression gate;
  # build-example above only catches link-time errors.
  #
  # Memory values are read from each example's Justfile at runtime
  # (the `memory` variable), so the matrix only carries args/expect.
  runtime-test:
    runs-on: ubuntu-latest
    needs: build-example
    permissions:
      contents: read
      packages: read
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        include:
          - example: helloworld-c
            args: ""
            expect: "Hello from C on Hyperlight"
          - example: rust
            args: ""
            expect: "Hello from Rust on Hyperlight"
          - example: go
            args: ""
            expect: "Hello from Go on Hyperlight"
          - example: shell
            args: "-- /demo.sh"
            expect: "Hello from Shell on Hyperlight"
          - example: python
            args: "-- /hello.py"
            expect: "Hello from Python on Hyperlight"
          - example: python-tools
            args: "-- /test_tools.py"
            expect: "Tool returned"
            needs_echo_tool: true
          - example: nodejs
            args: "-- /app/hello.js"
            expect: "Hello from Node.js on Hyperlight"
          - example: dotnet
            args: ""
            expect: "Hello, World! From .NET on Hyperlight"
          - example: dotnet-nativeaot
            args: ""
            expect: "Hello, World! From .NET NativeAOT on Hyperlight"
          - example: hostfs-posix-c
            args: ""
            expect: "done\\."
            needs_mount: true
          - example: hostfs-posix-py
            args: "-- /hostfs_demo.py"
            expect: "done\\."
            needs_mount: true
          - example: multifn-c
            args: ""
            expect: "RUN: world"
            driver: multifn-test
          - example: python-agent
            args: "-- /agent.py"
            expect: "done\\."
            needs_mount: true
          - example: python-agent-driver
            args: ""
            expect: "hello from driver"
            driver: pydriver-run
          - example: powershell
            args: "-- -NoProfile -File /scripts/hello.ps1"
            expect: "Hello, World! From PowerShell on Hyperlight"
          - example: networking-py
            args: "--net -- /urllib_get.py"
            expect: "SUCCESS: urllib GET worked!"
          - example: networking-py
            args: "--net -- /urllib_get_no_timeout.py"
            expect: "SUCCESS: urllib GET \\(no timeout\\) worked!"
          - example: networking-py
            args: "--port 8080 -- /echo_server_test.py"
            expect: "SUCCESS: bind\\+listen on port 8080 allowed"
          - example: go-http
            args: "--port 8080 -- /bin/server"
            expect: "Hello from Hyperlight-Unikraft!"
            http_port: "8080"
          - example: dotnet-http
            args: "--port 8080 -- /app/KestrelHyperlight"
            expect: "Hello from Kestrel on Hyperlight!"
            http_port: "8080"
    steps:
      - uses: actions/checkout@v4

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: host -> target

      - name: Enable KVM permissions (no-op if device absent)
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm || true

      - name: Check KVM availability
        id: kvm_check
        run: |
          if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then
            echo "available=true" >> $GITHUB_OUTPUT
            ls -la /dev/kvm
          else
            echo "available=false" >> $GITHUB_OUTPUT
            echo "::warning::/dev/kvm is not available on this runner; runtime test skipped"
          fi

      - name: Install Go (for kraft-hyperlight)
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          curl -sL https://go.dev/dl/go1.25.1.linux-amd64.tar.gz | sudo tar -C /usr/local -xz
          echo "/usr/local/go/bin" >> $GITHUB_PATH

      - name: Install just
        if: steps.kvm_check.outputs.available == 'true'
        uses: extractions/setup-just@v2

      - name: Build kraft-hyperlight
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          git clone --branch hyperlight-platform --depth 1 \
              https://github.com/danbugs/kraftkit.git /tmp/kraftkit
          cd /tmp/kraftkit && go build -o kraft-hyperlight ./cmd/kraft
          sudo mv kraft-hyperlight /usr/local/bin/

      - name: Build host binaries
        if: steps.kvm_check.outputs.available == 'true'
        run: |
          cd host
          cargo build --release --features wasm-host-fns --bin hyperlight-unikraft --bin multifn-test --bin pydriver-run
          sudo cp target/release/hyperlight-unikraft /usr/local/bin/

      - name: Build echo Wasm host function
        if: steps.kvm_check.outputs.available == 'true' && matrix.needs_echo_tool == true
        run: |
          rustup target add wasm32-wasip1
          cargo build --manifest-path examples/echo-wasm-host-fxn/Cargo.toml --release --target wasm32-wasip1

      - name: Build local-python-base images (python-agent-driver only)
        if: steps.kvm_check.outputs.available == 'true' && matrix.example == 'python-agent-driver'
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          docker build --target base -t local-python-base-dev:latest \
              -f runtimes/python.Dockerfile runtimes/
          docker build -t local-python-base:latest \
              -f runtimes/python.Dockerfile runtimes/

      - name: Build rootfs + kernel
        if: steps.kvm_check.outputs.available == 'true'
        working-directory: examples/${{ matrix.example }}
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          just rootfs
          kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 || true
          if ! ls .unikraft/build/*_hyperlight-x86_64 >/dev/null 2>&1; then
            echo "::warning::kraft build produced no kernel; cloning manually and rebuilding"
            UK_SOURCE=$(awk '/^unikraft:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            UK_BRANCH=$(awk '/^unikraft:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            ELF_SOURCE=$(awk '/app-elfloader:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            ELF_BRANCH=$(awk '/app-elfloader:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            mkdir -p .unikraft/apps .unikraft/libs
            rm -rf .unikraft/unikraft .unikraft/apps/elfloader .unikraft/libs/libelf .unikraft/build
            git clone --branch "$UK_BRANCH" --depth 1 "$UK_SOURCE" .unikraft/unikraft
            git clone --branch "$ELF_BRANCH" --depth 1 "$ELF_SOURCE" .unikraft/apps/elfloader
            git clone --branch staging --depth 1 https://github.com/unikraft/lib-libelf.git .unikraft/libs/libelf
            kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64
          fi

      - name: Prepare pydriver-run script (python-agent-driver only)
        if: steps.kvm_check.outputs.available == 'true' && matrix.driver == 'pydriver-run'
        run: echo 'print("hello from driver")' > /tmp/tiny.py

      - name: Run and check output
        if: steps.kvm_check.outputs.available == 'true'
        working-directory: examples/${{ matrix.example }}
        run: |
          # Strict mode: expected output must appear AND the driver
          # process must exit cleanly. A guest that prints the right
          # banner and then crashes no longer counts as passing.
          # NOTE: `set -o pipefail` is scoped to the driver invocation
          # below, not applied globally — otherwise the artifact-lookup
          # pipelines fail when one of the glob patterns doesn't match.
          kernel=$(ls .unikraft/build/*_hyperlight-x86_64 2>/dev/null | head -1 || true)
          cpio=$(ls *-initrd.cpio initrd.cpio 2>/dev/null | head -1 || true)
          if [ -z "$kernel" ] || [ -z "$cpio" ]; then
            echo "FAIL: missing kernel ($kernel) or initrd ($cpio)"
            ls -la .unikraft/build/ *.cpio 2>&1 | head -40 || true
            exit 1
          fi
          # Read memory from the example's Justfile (single source of truth).
          memory=$(grep '^memory' Justfile | sed 's/memory.*:= *"//' | sed 's/".*//')
          if [ -z "$memory" ]; then
            echo "::notice::No memory variable in Justfile, using binary default"
          fi
          expect='${{ matrix.expect }}'
          mount_args=""
          if [ "${{ matrix.needs_mount }}" = "true" ]; then
            mount_dir="$RUNNER_TEMP/hostfs-${{ matrix.example }}"
            mkdir -p "$mount_dir"
            mount_args="--mount $mount_dir:/host"
          fi

          tool_args=""
          if [ "${{ matrix.needs_echo_tool }}" = "true" ]; then
            tool_args="--tool echo=../echo-wasm-host-fxn/target/wasm32-wasip1/release/echo-wasm-host-fxn.wasm"
          fi

          # HTTP server examples: start in background, poll, curl, kill.
          http_port="${{ matrix.http_port }}"
          if [ -n "$http_port" ]; then
            mem_args=""
            if [ -n "$memory" ]; then
              mem_args="-m $memory"
            fi
            hyperlight-unikraft -q $mem_args "$kernel" --initrd "$cpio" $mount_args $tool_args ${{ matrix.args }} &
            server_pid=$!
            sleep 3
            ready=0
            for i in $(seq 1 30); do
              if curl -s --max-time 10 "http://127.0.0.1:${http_port}" >/dev/null 2>&1; then
                ready=1
                break
              fi
              sleep 1
            done
            if [ "$ready" -ne 1 ]; then
              echo "FAIL: server did not become ready within timeout"
              kill "$server_pid" 2>/dev/null || true
              exit 1
            fi
            response=$(curl -s --max-time 10 "http://127.0.0.1:${http_port}")
            kill "$server_pid" 2>/dev/null || true
            wait "$server_pid" 2>/dev/null || true
            echo "=== HTTP response ==="
            echo "$response"
            if echo "$response" | grep -qF "$expect"; then
              echo "PASS: matched /$expect/"
            else
              echo "FAIL: did not match /$expect/"
              exit 1
            fi
            exit 0
          fi
          case "${{ matrix.driver }}" in
            multifn-test)
              cmd=(timeout 60 /home/runner/work/hyperlight-unikraft/hyperlight-unikraft/host/target/release/multifn-test "$kernel" "$cpio")
              ;;
            pydriver-run)
              cmd=(timeout 120 /home/runner/work/hyperlight-unikraft/hyperlight-unikraft/host/target/release/pydriver-run "$kernel" "$cpio" /tmp/tiny.py)
              ;;
            *)
              mem_args=""
              if [ -n "$memory" ]; then
                mem_args="-m $memory"
              fi
              cmd=(timeout 120 hyperlight-unikraft -q $mem_args "$kernel" --initrd "$cpio" $mount_args $tool_args ${{ matrix.args }})
              ;;
          esac
          set +e
          output=$("${cmd[@]}" 2>&1)
          exit_code=$?
          set -e
          output=$(printf '%s' "$output" | tr -d '\0')
          echo "=== output (exit=$exit_code) ==="
          echo "$output" | head -40
          matched=0
          echo "$output" | grep -qE "$expect" && matched=1
          if [ "$matched" -ne 1 ]; then
            echo "FAIL: did not match /$expect/"
            exit 1
          fi
          if [ "$exit_code" -ne 0 ]; then
            echo "FAIL: driver exited with $exit_code after printing expected output"
            exit 1
          fi
          echo "PASS: matched /$expect/ and exited cleanly"

  # Build each example's kernel+initrd on Linux once and upload as a
  # per-example artifact. Matches the Windows runtime matrix so Windows
  # gets the same example coverage without needing kraft-hyperlight or
  # a Linux docker toolchain on Windows.
  package-images-for-windows:
    runs-on: ubuntu-latest
    needs: build-example
    permissions:
      contents: read
      packages: read
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        example:
          - helloworld-c
          - rust
          - go
          - shell
          - python
          - python-tools
          - nodejs
          - dotnet
          - dotnet-nativeaot
          - hostfs-posix-c
          - hostfs-posix-py
          - multifn-c
          - python-agent
          - python-agent-driver
          - powershell
          - networking-py
          - go-http
          - dotnet-http
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-go@v5
        with:
          go-version: "1.25.1"
          cache: false

      - name: Install just
        uses: extractions/setup-just@v2

      - name: Cache kraft-hyperlight
        id: kraft-cache
        uses: actions/cache@v4
        with:
          path: /usr/local/bin/kraft-hyperlight
          key: kraft-hyperlight-linux-${{ hashFiles('.github/workflows/test-examples.yml') }}

      - name: Build kraft-hyperlight
        if: steps.kraft-cache.outputs.cache-hit != 'true'
        run: |
          git clone --branch hyperlight-platform --depth 1 \
              https://github.com/danbugs/kraftkit.git /tmp/kraftkit
          cd /tmp/kraftkit && go build -o kraft-hyperlight ./cmd/kraft
          sudo mv kraft-hyperlight /usr/local/bin/

      - name: Build local-python-base images (python-agent-driver only)
        if: matrix.example == 'python-agent-driver'
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          docker build --target base -t local-python-base-dev:latest \
              -f runtimes/python.Dockerfile runtimes/
          docker build -t local-python-base:latest \
              -f runtimes/python.Dockerfile runtimes/

      - name: Build rootfs + kernel
        working-directory: examples/${{ matrix.example }}
        env:
          DOCKER_BUILDKIT: "0"
        run: |
          just rootfs
          kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64 || true
          if ! ls .unikraft/build/*_hyperlight-x86_64 >/dev/null 2>&1; then
            echo "::warning::kraft build produced no kernel; cloning manually and rebuilding"
            UK_SOURCE=$(awk '/^unikraft:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            UK_BRANCH=$(awk '/^unikraft:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            ELF_SOURCE=$(awk '/app-elfloader:/{f=1} f && /source:/{print $2; exit}' kraft.yaml)
            ELF_BRANCH=$(awk '/app-elfloader:/{f=1} f && /version:/{print $2; exit}' kraft.yaml)
            mkdir -p .unikraft/apps .unikraft/libs
            rm -rf .unikraft/unikraft .unikraft/apps/elfloader .unikraft/libs/libelf .unikraft/build
            git clone --branch "$UK_BRANCH" --depth 1 "$UK_SOURCE" .unikraft/unikraft
            git clone --branch "$ELF_BRANCH" --depth 1 "$ELF_SOURCE" .unikraft/apps/elfloader
            git clone --branch staging --depth 1 https://github.com/unikraft/lib-libelf.git .unikraft/libs/libelf
            kraft-hyperlight --no-prompt build --plat hyperlight --arch x86_64
          fi

      - name: Package image
        working-directory: examples/${{ matrix.example }}
        run: |
          mkdir -p /tmp/image
          cp .unikraft/build/*_hyperlight-x86_64 /tmp/image/kernel
          cpio_file=$(ls *-initrd.cpio 2>/dev/null | head -1)
          if [ -z "$cpio_file" ]; then
            cpio_file=$(ls initrd.cpio 2>/dev/null | head -1)
          fi
          cp "$cpio_file" /tmp/image/initrd.cpio
          ls -la /tmp/image/

      - name: Upload image artifact
        uses: actions/upload-artifact@v4
        with:
          name: windows-image-${{ matrix.example }}
          path: /tmp/image/
          retention-days: 1
          if-no-files-found: error

  # Windows runtime — 1:1 with Linux runtime-test: same examples, same
  # expected output patterns, same drivers (multifn-test, pydriver-run).
  # The only difference is we can't build the images on Windows (no
  # kraft-hyperlight + no Linux docker), so we download them from the
  # package-images-for-windows job.
  #
  # Memory values are read from each example's Justfile at runtime.
  runtime-test-windows:
    runs-on: windows-latest
    needs: package-images-for-windows
    permissions:
      contents: read
      packages: read
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - example: helloworld-c
            args: ""
            expect: "Hello from C on Hyperlight"
          - example: rust
            args: ""
            expect: "Hello from Rust on Hyperlight"
          - example: go
            args: ""
            expect: "Hello from Go on Hyperlight"
          - example: shell
            args: "-- /demo.sh"
            expect: "Hello from Shell on Hyperlight"
          - example: python
            args: "-- /hello.py"
            expect: "Hello from Python on Hyperlight"
          - example: python-tools
            args: "-- /test_tools.py"
            expect: "Tool returned"
            needs_echo_tool: true
          - example: nodejs
            args: "-- /app/hello.js"
            expect: "Hello from Node.js on Hyperlight"
          - example: dotnet
            args: ""
            expect: "Hello, World! From .NET on Hyperlight"
          - example: dotnet-nativeaot
            args: ""
            expect: "Hello, World! From .NET NativeAOT on Hyperlight"
          - example: hostfs-posix-c
            args: ""
            expect: "done\\."
            needs_mount: true
          - example: hostfs-posix-py
            args: "-- /hostfs_demo.py"
            expect: "done\\."
            needs_mount: true
          - example: multifn-c
            args: ""
            expect: "RUN: world"
            driver: multifn-test
          - example: python-agent
            args: "-- /agent.py"
            expect: "done\\."
            needs_mount: true
          - example: python-agent-driver
            args: ""
            expect: "hello from driver"
            driver: pydriver-run
          - example: powershell
            args: "-- -NoProfile -File /scripts/hello.ps1"
            expect: "Hello, World! From PowerShell on Hyperlight"
          - example: networking-py
            args: "--net -- /urllib_get.py"
            expect: "SUCCESS: urllib GET worked!"
          - example: networking-py
            args: "--net -- /urllib_get_no_timeout.py"
            expect: "SUCCESS: urllib GET \\(no timeout\\) worked!"
          - example: networking-py
            args: "--port 8080 -- /echo_server_test.py"
            expect: "SUCCESS: bind\\+listen on port 8080 allowed"
          - example: go-http
            args: "--port 8080 -- /bin/server"
            expect: "Hello from Hyperlight-Unikraft!"
            http_port: "8080"
          - example: dotnet-http
            args: "--port 8080 -- /app/KestrelHyperlight"
            expect: "Hello from Kestrel on Hyperlight!"
            http_port: "8080"
    steps:
      - uses: actions/checkout@v4

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: host -> target

      - name: Ensure surrogate build consistency
        shell: pwsh
        run: |
          $hlsExe = "host\target\release\build\hyperlight-host-*\out\..\..\hls\x86_64-pc-windows-msvc\release\hyperlight_surrogate.exe"
          if (-not (Resolve-Path $hlsExe -ErrorAction SilentlyContinue)) {
            Write-Host "Surrogate missing — clearing hyperlight-host fingerprints to force rebuild"
            Get-ChildItem "host\target\release\.fingerprint" -Filter "hyperlight-host-*" -Directory -ErrorAction SilentlyContinue |
              Remove-Item -Recurse -Force
          }

      - name: Build host binaries
        shell: pwsh
        run: |
          cd host
          cargo build --release --features wasm-host-fns --bin hyperlight-unikraft --bin multifn-test --bin pydriver-run --bin pyhl
          Copy-Item target\release\hyperlight-unikraft.exe $env:USERPROFILE\.cargo\bin\ -Force
          Copy-Item target\release\multifn-test.exe $env:USERPROFILE\.cargo\bin\ -Force
          Copy-Item target\release\pydriver-run.exe $env:USERPROFILE\.cargo\bin\ -Force
          Copy-Item target\release\pyhl.exe $env:USERPROFILE\.cargo\bin\ -Force

      - name: Build echo Wasm host function
        if: matrix.needs_echo_tool == true
        shell: pwsh
        run: |
          rustup target add wasm32-wasip1
          cargo build --manifest-path examples/echo-wasm-host-fxn/Cargo.toml --release --target wasm32-wasip1

      - name: Download prebuilt image
        uses: actions/download-artifact@v4
        with:
          name: windows-image-${{ matrix.example }}
          path: image

      - name: Run and check output
        shell: pwsh
        run: |
          $kernel = (Resolve-Path "image/kernel").Path
          $cpio   = (Resolve-Path "image/initrd.cpio").Path
          $expect = '${{ matrix.expect }}'
          $driver = '${{ matrix.driver }}'
          $runArgs = '${{ matrix.args }}'
          $needsMount = '${{ matrix.needs_mount }}'
          $needsEchoTool = '${{ matrix.needs_echo_tool }}'

          # Read memory from the example's Justfile (single source of truth).
          $memory = ''
          $justfile = "examples/${{ matrix.example }}/Justfile"
          if (Test-Path $justfile) {
            $memLine = Get-Content $justfile | Where-Object { $_ -match '^\s*memory\s*:=' }
            if ($memLine -match '"([^"]+)"') {
              $memory = $Matches[1]
            }
          }

          $mountArgs = @()
          if ($needsMount -eq 'true') {
            $mountDir = Join-Path $env:RUNNER_TEMP ('hostfs-' + '${{ matrix.example }}')
            New-Item -ItemType Directory -Force -Path $mountDir | Out-Null
            # Preopen::parse_cli splits on the last ':' and treats the right
            # side as the guest path iff it starts with '/', so Windows
            # drive-letter colons in $mountDir are handled correctly.
            $mountArgs = @('--mount', ($mountDir + ':/host'))
          }

          $toolArgs = @()
          if ($needsEchoTool -eq 'true') {
            $toolArgs = @('--tool', 'echo=examples/echo-wasm-host-fxn/target/wasm32-wasip1/release/echo-wasm-host-fxn.wasm')
          }

          # Strict mode: expected output must appear AND the driver
          # must exit 0. Matches Linux semantics.
          $PSNativeCommandUseErrorActionPreference = $false
          $ErrorActionPreference = 'Continue'

          # HTTP server examples: start in background, poll, curl, kill.
          $httpPort = '${{ matrix.http_port }}'
          if ($httpPort -ne '') {
            $argList = @()
            if ($runArgs -ne '') {
              $argList = $runArgs.Split(' ') | Where-Object { $_ -ne '' }
            }
            $memArgs = @()
            if ($memory -ne '') {
              $memArgs = @('-m', $memory)
            }
            $stderrLog = Join-Path $env:RUNNER_TEMP 'hl-stderr.log'
            $proc = Start-Process -FilePath 'hyperlight-unikraft' `
              -ArgumentList (@('-q') + $memArgs + @($kernel, '--initrd', $cpio) + $argList) `
              -PassThru -NoNewWindow -RedirectStandardError $stderrLog
            # Give the server time to boot the unikernel and start listening
            Start-Sleep -Seconds 5
            $ready = $false
            for ($i = 0; $i -lt 30; $i++) {
              try {
                $null = Invoke-WebRequest -Uri "http://127.0.0.1:${httpPort}" -UseBasicParsing -TimeoutSec 10 -ErrorAction Stop
                $ready = $true
                break
              } catch {
                Start-Sleep -Seconds 1
              }
            }
            if (-not $ready) {
              Write-Host "FAIL: server did not become ready within timeout"
              Write-Host "=== stderr (last 200 lines) ==="
              Get-Content $stderrLog -Tail 200 -ErrorAction SilentlyContinue
              Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
              exit 1
            }
            $raw = (Invoke-WebRequest -Uri "http://127.0.0.1:${httpPort}" -UseBasicParsing -TimeoutSec 10).Content
            if ($raw -is [byte[]]) { $resp = [System.Text.Encoding]::UTF8.GetString($raw) } else { $resp = $raw }
            Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
            Write-Host "=== HTTP response ==="
            Write-Host $resp
            if ($resp -match [regex]::Escape($expect)) {
              Write-Host "PASS: matched /$expect/"
              exit 0
            } else {
              Write-Host "FAIL: did not match /$expect/"
              exit 1
            }
          }

          # Helper: run a command with a per-test timeout (seconds).
          function Invoke-WithTimeout {
            param([int]$Seconds, [string]$Exe, [string[]]$ArgList)
            $outFile = Join-Path $env:RUNNER_TEMP 'hl-test-out.log'
            $proc = Start-Process -FilePath $Exe -ArgumentList $ArgList `
              -NoNewWindow -RedirectStandardOutput $outFile -RedirectStandardError "$outFile.err" -PassThru
            if (-not $proc.WaitForExit($Seconds * 1000)) {
              Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue
              Write-Host "FAIL: timed out after ${Seconds}s"
              Get-Content $outFile -ErrorAction SilentlyContinue
              Get-Content "$outFile.err" -ErrorAction SilentlyContinue
              exit 1
            }
            $stdoutContent = Get-Content $outFile -Raw -ErrorAction SilentlyContinue
            $stderrContent = Get-Content "$outFile.err" -Raw -ErrorAction SilentlyContinue
            $script:out = "$stdoutContent$stderrContent".Trim()
            $script:rc = $proc.ExitCode
          }

          switch ($driver) {
            'multifn-test' {
              Invoke-WithTimeout -Seconds 60 -Exe 'multifn-test' -ArgList @($kernel, $cpio)
            }
            'pydriver-run' {
              "print('hello from driver')" | Out-File -Encoding ascii tiny.py
              $tiny = (Resolve-Path "tiny.py").Path
              Invoke-WithTimeout -Seconds 120 -Exe 'pydriver-run' -ArgList @($kernel, $cpio, $tiny)
            }
            default {
              $argList = @()
              if ($runArgs -ne '') {
                $argList = $runArgs.Split(' ') | Where-Object { $_ -ne '' }
              }
              $memArgs = @()
              if ($memory -ne '') {
                $memArgs = @('-m', $memory)
              }
              Invoke-WithTimeout -Seconds 120 -Exe 'hyperlight-unikraft' `
                -ArgList (@('-q') + $memArgs + @($kernel, '--initrd', $cpio) + $mountArgs + $toolArgs + $argList)
            }
          }

          if ($null -eq $out) { $out = '' }
          Write-Host "=== output (exit=$rc) ==="
          Write-Host $out
          if (-not ($out -match $expect)) {
            Write-Host "FAIL: did not match /$expect/"
            exit 1
          }
          if ($rc -ne 0) {
            Write-Host "FAIL: driver exited with $rc after printing expected output"
            exit 1
          }
          Write-Host "PASS: matched /$expect/ and exited cleanly"
          exit 0

  # pyhl snapshot-to-disk end-to-end — runs the exact same flow on both
  # Linux and Windows:
  #   1. pyhl setup --from <src-dir>   (warmup + persist snapshot)
  #   2. pyhl run hi.py                (hello)
  #   3. pyhl run pandas_test.py       (pandas — exercises the warm snapshot)
  #   4. pyhl run --repeat 4 pandas    (hermetic rewind, 5 iterations)
  # Both OSes consume the python-agent-driver image built on Linux in
  # package-images-for-windows. This is the real CoW/restore regression
  # gate — MAP_PRIVATE on Linux and PAGE_WRITECOPY+FILE_MAP_COPY on
  # Windows both have to stay working.
  pyhl-snapshot-test:
    needs: package-images-for-windows
    permissions:
      contents: read
      packages: read
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: host -> target

      - name: Ensure surrogate build consistency (Windows only)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          $hlsExe = "host\target\release\build\hyperlight-host-*\out\..\..\hls\x86_64-pc-windows-msvc\release\hyperlight_surrogate.exe"
          if (-not (Resolve-Path $hlsExe -ErrorAction SilentlyContinue)) {
            Write-Host "Surrogate missing — clearing hyperlight-host fingerprints to force rebuild"
            Get-ChildItem "host\target\release\.fingerprint" -Filter "hyperlight-host-*" -Directory -ErrorAction SilentlyContinue |
              Remove-Item -Recurse -Force
          }

      # Linux needs /dev/kvm access to run the guest.
      - name: Enable KVM permissions (Linux only)
        if: runner.os == 'Linux'
        run: |
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm || true

      - name: Check KVM availability (Linux only)
        if: runner.os == 'Linux'
        id: kvm_check
        run: |
          if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then
            echo "available=true" >> $GITHUB_OUTPUT
            ls -la /dev/kvm
          else
            echo "available=false" >> $GITHUB_OUTPUT
            echo "::warning::/dev/kvm is not available; pyhl test skipped"
          fi

      - name: Install pyhl (Linux)
        if: runner.os == 'Linux'
        run: |
          cd host
          cargo build --release --bin pyhl
          sudo cp target/release/pyhl /usr/local/bin/

      - name: Install pyhl (Windows)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          cd host
          cargo build --release --bin pyhl
          Copy-Item target\release\pyhl.exe $env:USERPROFILE\.cargo\bin\ -Force

      - name: Download prebuilt python-agent-driver image
        uses: actions/download-artifact@v4
        with:
          name: windows-image-python-agent-driver
          path: pyhl-image

      # `pyhl setup --from` expects a build-tree shape:
      #   <dir>/.unikraft/build/*_hyperlight-x86_64
      #   <dir>/*-initrd.cpio
      - name: Lay out pyhl setup source dir (Linux)
        if: runner.os == 'Linux' && steps.kvm_check.outputs.available == 'true'
        run: |
          mkdir -p src-dir/.unikraft/build
          mv pyhl-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64
          mv pyhl-image/initrd.cpio src-dir/pyhl-initrd.cpio
          ls -laR src-dir

      - name: Lay out pyhl setup source dir (Windows)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          New-Item -ItemType Directory -Force -Path src-dir/.unikraft/build | Out-Null
          Move-Item pyhl-image/kernel src-dir/.unikraft/build/pyhl-kernel_hyperlight-x86_64
          Move-Item pyhl-image/initrd.cpio src-dir/pyhl-initrd.cpio
          Get-ChildItem -Recurse src-dir

      - name: pyhl setup --from (warms up + persists snapshot)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: pyhl setup --from src-dir --force

      - name: pyhl run (hello)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: |
          "print('hello from pyhl CI')" | Out-File -Encoding ascii hi.py
          $out = pyhl run -v hi.py 2>&1
          Write-Host $out
          if ($out -match 'hello from pyhl CI') {
            Write-Host "PASS: matched /hello from pyhl CI/"
          } else {
            Write-Error "FAIL: did not match /hello from pyhl CI/"
            exit 1
          }

      - name: pyhl run (pandas — exercises the warm snapshot)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: |
          @"
          import pandas as pd, numpy as np
          df = pd.DataFrame({'x': np.arange(5)})
          print('pandas_ok:', df.sum().to_dict())
          "@ | Out-File -Encoding ascii pandas_test.py
          $out = pyhl run -v pandas_test.py 2>&1
          Write-Host $out
          if ($out -match 'pandas_ok:') {
            Write-Host "PASS: matched /pandas_ok:/"
          } else {
            Write-Error "FAIL: did not match /pandas_ok:/"
            exit 1
          }

      - name: pyhl run --repeat 4 (exercise hermetic rewind)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: |
          $out = pyhl run -v --repeat 4 pandas_test.py 2>&1
          Write-Host $out
          # Expect the pandas output five times (run 1/5 .. run 5/5)
          $hits = ($out | Select-String -Pattern 'pandas_ok:' -AllMatches).Matches.Count
          if ($hits -ge 5) {
            Write-Host "PASS: --repeat produced $hits pandas_ok lines"
          } else {
            Write-Error "FAIL: expected >=5 pandas_ok lines, got $hits"
            exit 1
          }

      - name: pyhl run (busybox subprocess demo)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: |
          $out = pyhl run examples/python-agent-driver/demo_busybox.py 2>&1
          Write-Host $out
          if ($out -match 'hello from hyperlight guest') {
            Write-Host "PASS: busybox subprocess demo"
          } else {
            Write-Error "FAIL: did not match /hello from hyperlight guest/"
            exit 1
          }

      - name: pyhl run (pip install subprocess demo)
        if: runner.os == 'Windows' || steps.kvm_check.outputs.available == 'true'
        shell: pwsh
        run: |
          $out = pyhl run --net examples/python-agent-driver/demo_pip_install.py 2>&1
          Write-Host $out
          if ($out -match 'Installed and imported six') {
            Write-Host "PASS: pip install subprocess demo"
          } else {
            Write-Error "FAIL: did not match /Installed and imported six/"
            exit 1
          }

  test-examples-passed:
    if: always()
    needs:
      [
        build-example,
        runtime-test,
        package-images-for-windows,
        runtime-test-windows,
        pyhl-snapshot-test,
      ]
    runs-on: ubuntu-latest
    permissions: {}
    steps:
      - run: |
          # Every job must succeed. "skipped" means a dependency failed
          # and the job never ran — that must fail the gate, not sneak
          # through as green.
          declare -A results=(
            [build-example]="${{ needs.build-example.result }}"
            [runtime-test]="${{ needs.runtime-test.result }}"
            [package-images-for-windows]="${{ needs.package-images-for-windows.result }}"
            [runtime-test-windows]="${{ needs.runtime-test-windows.result }}"
            [pyhl-snapshot-test]="${{ needs.pyhl-snapshot-test.result }}"
          )
          failed=0
          for job in "${!results[@]}"; do
            r="${results[$job]}"
            if [[ "$r" != "success" ]]; then
              echo "FAIL: $job = $r"
              failed=1
            fi
          done
          if [[ "$failed" -eq 1 ]]; then
            exit 1
          fi
          echo "All checks passed"