run-benchmark.yml

name: Run Benchmark

on:
  push:
    branches: ["main"]
  pull_request:
  workflow_dispatch:
    inputs:
      instance_family:
        description: Instance family to use for the benchmark runner
        type: string
        required: false
        default: g6e.2xlarge
      mode:
        description: Host harness mode
        type: choice
        required: false
        default: prove-stark
        options:
          - execute
          - execute-metered
          - prove-app
          - prove-stark
      host_profile:
        description: Cargo profile for the host harness
        type: choice
        required: false
        default: release
        options:
          - dev
          - release
      iterations:
        description: Optional CoreMark ITERATIONS override
        type: string
        required: false
        default: ""

concurrency:
  group: ${{ github.workflow_ref }}-run-benchmark-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  CARGO_NET_GIT_FETCH_WITH_CLI: "true"

jobs:
  benchmark:
    name: Run Host Benchmark
    env:
      BENCH_HOST_PROFILE: ${{ inputs.host_profile || 'release' }}
      BENCH_ITERATIONS: ${{ inputs.iterations || '' }}
      BENCH_MODE: ${{ inputs.mode || 'prove-stark' }}
    runs-on:
      - runs-on
      - run-id=${{ github.run_id }}
      - family=${{ inputs.instance_family || 'g6e.2xlarge' }}
      - disk=large
      - tag=bench-coremark-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
      - extras=s3-cache
      - spot=false
      - image=${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') && 'ubuntu24-gpu-x64' || contains(inputs.instance_family || 'g6e.2xlarge', 'g.') && 'ubuntu24-full-arm64' || 'ubuntu24-full-x64' }}
    timeout-minutes: 180

    steps:
      - uses: runs-on/action@v2

      - name: Checkout repository
        uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@nightly
        env:
          RUSTUP_PERMIT_COPY_RENAME: "1"
        with:
          toolchain: nightly-2025-08-19
          components: clippy,rustfmt

      - name: Cache Rust artifacts
        uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y gnuplot

      - name: Verify GPU setup
        if: ${{ startsWith(inputs.instance_family || 'g6e.2xlarge', 'g') }}
        run: |
          nvidia-smi
          nvidia-smi -L

      - name: Resolve RISC-V GCC
        run: |
          candidates=(
            riscv32-unknown-elf-gcc
            riscv64-unknown-elf-gcc
            riscv32-linux-gnu-gcc
            riscv64-linux-gnu-gcc
            riscv-none-elf-gcc
            riscv64-unknown-linux-gnu-gcc
          )

          for cc in "${candidates[@]}"; do
            if command -v "$cc" >/dev/null 2>&1; then
              echo "OPENVM_GUEST_GCC=$cc" >> "$GITHUB_ENV"
              echo "Using $cc"
              exit 0
            fi
          done

          sudo apt-get update
          sudo apt-get install -y gcc-riscv64-unknown-elf

          if command -v riscv64-unknown-elf-gcc >/dev/null 2>&1; then
            echo "OPENVM_GUEST_GCC=riscv64-unknown-elf-gcc" >> "$GITHUB_ENV"
            echo "Installed and using riscv64-unknown-elf-gcc"
            exit 0
          fi

          echo "No supported RISC-V GCC toolchain was found." >&2
          exit 1

      - name: Checkout openvm
        run: git clone --depth=1 --branch="v2.0.0" https://github.com/openvm-org/openvm.git

      - name: Install cargo-openvm
        working-directory: openvm/crates/cli
        run: cargo install --force --locked --profile=release --path .

      - name: Install openvm-prof
        working-directory: openvm/crates/prof
        run: cargo install --force --profile=dev --path .

      - name: Build and stage guest ELF
        run: |
          mkdir -p host/elf

          if [[ -n "$BENCH_ITERATIONS" ]]; then
            CFLAGS="-DITERATIONS=$BENCH_ITERATIONS" cargo openvm build
          else
            cargo openvm build
          fi

          cp target/riscv32im-risc0-zkvm-elf/release/openvm-coremark host/elf/openvm-coremark

      - name: Run host harness benchmark
        run: |
          ./host/scripts/run_coremark.sh \
            --mode "$BENCH_MODE" \
            --profile "$BENCH_HOST_PROFILE"

      - name: Generate openvm-prof outputs
        run: |
          openvm-prof --json-paths metrics.json --output-json openvm-prof.json

      - name: Add benchmark summary
        if: always()
        run: |
          echo "## Host Benchmark" >> "$GITHUB_STEP_SUMMARY"
          echo "- Mode: \`$BENCH_MODE\`" >> "$GITHUB_STEP_SUMMARY"
          echo "- Host profile: \`$BENCH_HOST_PROFILE\`" >> "$GITHUB_STEP_SUMMARY"
          if [[ -n "$BENCH_ITERATIONS" ]]; then
            echo "- ITERATIONS override: \`$BENCH_ITERATIONS\`" >> "$GITHUB_STEP_SUMMARY"
          fi
          echo "" >> "$GITHUB_STEP_SUMMARY"

          if [[ -f metrics.md ]]; then
            cat metrics.md >> "$GITHUB_STEP_SUMMARY"
          elif [[ -f metrics.json ]]; then
            echo "Generated \`metrics.json\`, but no markdown summary was found." >> "$GITHUB_STEP_SUMMARY"
          else
            echo "No benchmark metrics were generated." >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload benchmark artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: host-benchmark-${{ inputs.mode || 'prove-stark' }}-${{ github.run_id }}
          if-no-files-found: warn
          path: |
            metrics.json
            metrics.md
            metrics.detailed.md
            openvm-prof.json
            gpu_memory_usage.csv