bench.yml

# Benchmark CI workflow
# Runs on pushes to main and PRs to detect performance regressions
name: Benchmarks

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:  # Allow manual triggering

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: -D warnings

jobs:
  benchmark:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: pi_agent_rust
        shell: bash
    steps:
      - name: Free disk space
        working-directory: /
        run: |
          set -euxo pipefail
          df -h /
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
            /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go \
            /opt/hostedtoolcache/Python /opt/hostedtoolcache/Ruby \
            /opt/hostedtoolcache/Java_Temurin-Hotspot_jdk \
            /usr/local/share/powershell /usr/share/swift \
            /usr/local/graalvm /usr/local/.ghcup /usr/local/julia*
          sudo docker image prune --all --force || true
          sudo apt-get clean || true
          df -h /

      - name: Checkout pi_agent_rust
        uses: actions/checkout@v4
        with:
          path: pi_agent_rust

      - name: Install system deps (bc, fd, rg, xcb) [linux]
        run: |
          set -euxo pipefail
          sudo apt-get update
          sudo apt-get install -y bc fd-find ripgrep libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
          sudo ln -sf "$(command -v fdfind)" /usr/local/bin/fd

      - name: Install Rust nightly
        uses: dtolnay/rust-toolchain@nightly
        with:
          components: clippy

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
            pi_agent_rust/target
          key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-bench-
            ${{ runner.os }}-cargo-

      - name: Standardize benchmark environment
        run: |
          set -euxo pipefail
          # Apply benchmark-optimal OS settings (best-effort, some may fail in containers)
          sudo scripts/bench_env_setup.sh apply || echo "::warning::Could not fully apply bench env (expected in containers)"
          # Emit environment fingerprint for artifact tracking
          mkdir -p target/perf
          scripts/bench_env_setup.sh fingerprint | tee target/perf/bench_env_fingerprint.json || true
          # Validate and report
          echo "## Benchmark Environment" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          scripts/bench_env_setup.sh validate 2>&1 | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY || true
          echo '```' >> $GITHUB_STEP_SUMMARY

      - name: Build release binary
        run: cargo build --release

      - name: Check binary size
        run: |
          SIZE_BYTES=$(stat --printf="%s" target/release/pi)
          SIZE_MB=$(echo "scale=2; $SIZE_BYTES / 1024 / 1024" | bc)
          echo "## Binary Size" >> $GITHUB_STEP_SUMMARY
          echo "- Size: ${SIZE_MB}MB" >> $GITHUB_STEP_SUMMARY
          echo "- Budget: 20MB" >> $GITHUB_STEP_SUMMARY
          if (( $(echo "$SIZE_MB > 20" | bc -l) )); then
            echo "::error::Binary size ${SIZE_MB}MB exceeds 20MB budget"
            exit 1
          fi
          echo "- Status: ✅ Within budget" >> $GITHUB_STEP_SUMMARY

      - name: Run micro-benchmarks (tools)
        run: cargo bench --bench tools -- --noplot

      - name: Run micro-benchmarks (extensions)
        run: cargo bench --bench extensions -- --noplot

      - name: Run system benchmarks
        run: cargo bench --bench system -- --noplot
        env:
          PI_BENCH_BINARY: target/release/pi

      - name: Run TUI performance benchmarks (PERF-8/9)
        run: cargo bench --bench tui_perf -- --noplot --save-baseline pr

      - name: Generate PiJS workload perf data (JSONL)
        run: |
          set -euxo pipefail
          PERF_PROFILE=perf
          mkdir -p "target/perf/${PERF_PROFILE}"
          PI_BENCH_BUILD_PROFILE="${PERF_PROFILE}" cargo run --profile "${PERF_PROFILE}" --bin pijs_workload -- --iterations 2000 --tool-calls 1 > "target/perf/${PERF_PROFILE}/pijs_workload_${PERF_PROFILE}.jsonl"
          PI_BENCH_BUILD_PROFILE="${PERF_PROFILE}" cargo run --profile "${PERF_PROFILE}" --bin pijs_workload -- --iterations 2000 --tool-calls 10 >> "target/perf/${PERF_PROFILE}/pijs_workload_${PERF_PROFILE}.jsonl"

      - name: Generate benchmark summary
        run: |
          echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Startup Time" >> $GITHUB_STEP_SUMMARY
          if [ -d "target/criterion/startup" ]; then
            for dir in target/criterion/startup/*/; do
              name=$(basename "$dir")
              if [ -f "$dir/new/estimates.json" ]; then
                mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
                echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
              fi
            done
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Truncation" >> $GITHUB_STEP_SUMMARY
          if [ -d "target/criterion/truncation" ]; then
            for dir in target/criterion/truncation/*/; do
              name=$(basename "$dir")
              if [ -f "$dir/new/estimates.json" ]; then
                mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
                echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
              fi
            done
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### TUI Rendering (PERF-8)" >> $GITHUB_STEP_SUMMARY
          for group in build_conversation_content view viewport_operations markdown_rendering; do
            if [ -d "target/criterion/$group" ]; then
              echo "**$group**" >> $GITHUB_STEP_SUMMARY
              for dir in target/criterion/"$group"/*/; do
                name=$(basename "$dir")
                if [ -f "$dir/new/estimates.json" ]; then
                  mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
                  echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
                fi
              done
            fi
          done

      - name: Perf budget gate
        run: cargo test --test perf_budgets -- --nocapture

      - name: Restore benchmark environment
        if: always()
        run: sudo scripts/bench_env_setup.sh restore || true

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-${{ github.sha }}
          path: |
            pi_agent_rust/target/criterion/
            pi_agent_rust/target/perf/
          retention-days: 30

  # Compare with main branch baseline (only on PRs)
  compare:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    needs: benchmark
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4
        with:
          ref: main
          path: main-branch

      - uses: actions/checkout@v4
        with:
          path: pr-branch

      - name: Install system deps (xcb) [linux]
        run: |
          set -euxo pipefail
          sudo apt-get update
          sudo apt-get install -y libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev

      - name: Install Rust nightly
        uses: dtolnay/rust-toolchain@nightly

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-compare-${{ hashFiles('**/Cargo.lock') }}

      - name: Install critcmp
        run: cargo install critcmp

      - name: Build and benchmark main
        working-directory: main-branch
        run: |
          cargo build --release
          cargo bench --bench tools -- --noplot --save-baseline main 2>/dev/null || true
          cargo bench --bench tui_perf -- --noplot --save-baseline main 2>/dev/null || true

      - name: Copy main baseline to PR target dir
        run: |
          set -euxo pipefail
          if [ -d "main-branch/target/criterion" ]; then
            mkdir -p pr-branch/target
            mkdir -p pr-branch/target/criterion
            cp -R main-branch/target/criterion/. pr-branch/target/criterion/
          fi

      - name: Build and benchmark PR
        working-directory: pr-branch
        run: |
          cargo build --release
          cargo bench --bench tools -- --noplot --baseline main 2>/dev/null || echo "Baseline comparison skipped"
          cargo bench --bench tui_perf -- --noplot --baseline main 2>/dev/null || echo "TUI baseline comparison skipped"

      - name: Compare TUI performance (fail on >20% regression)
        working-directory: pr-branch
        env:
          PERF_REGRESSION_THRESHOLD: ${{ vars.PERF_REGRESSION_THRESHOLD || '20' }}
        run: |
          set -euo pipefail
          echo "## TUI Performance Comparison" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Run critcmp with threshold
          if critcmp main pr --threshold "${PERF_REGRESSION_THRESHOLD}" 2>/dev/null; then
            echo "✅ No TUI performance regressions detected (threshold: ${PERF_REGRESSION_THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
          else
            echo "❌ TUI performance regression detected (>${PERF_REGRESSION_THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
            # Show detailed comparison
            critcmp main pr 2>/dev/null >> $GITHUB_STEP_SUMMARY || true
            exit 1
          fi