Skip to content
Benchmarks

Merge remote-tracking branch 'upstream/main' #2

Sign in to view logs

Merge remote-tracking branch 'upstream/main'

Merge remote-tracking branch 'upstream/main' #2

Workflow file for this run

.github/workflows/bench.yml at 542c5b0
# Benchmark CI workflow
# Runs on pushes to main and PRs to detect performance regressions
name: Benchmarks
on:
push:
branches: [main]
pull_request:
branches: [main]
workflow_dispatch: # Allow manual triggering
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: -D warnings
jobs:
benchmark:
runs-on: ubuntu-latest
defaults:
run:
working-directory: pi_agent_rust
shell: bash
steps:
- name: Free disk space
working-directory: /
run: |
set -euxo pipefail
df -h /
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go \
/opt/hostedtoolcache/Python /opt/hostedtoolcache/Ruby \
/opt/hostedtoolcache/Java_Temurin-Hotspot_jdk \
/usr/local/share/powershell /usr/share/swift \
/usr/local/graalvm /usr/local/.ghcup /usr/local/julia*
sudo docker image prune --all --force || true
sudo apt-get clean || true
df -h /
- name: Checkout pi_agent_rust
uses: actions/checkout@v4
with:
path: pi_agent_rust
- name: Install system deps (bc, fd, rg, xcb) [linux]
run: |
set -euxo pipefail
sudo apt-get update
sudo apt-get install -y bc fd-find ripgrep libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
sudo ln -sf "$(command -v fdfind)" /usr/local/bin/fd
- name: Install Rust nightly
uses: dtolnay/rust-toolchain@nightly
with:
components: clippy
- name: Cache cargo registry
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
pi_agent_rust/target
key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-bench-
${{ runner.os }}-cargo-
- name: Standardize benchmark environment
run: |
set -euxo pipefail
# Apply benchmark-optimal OS settings (best-effort, some may fail in containers)
sudo scripts/bench_env_setup.sh apply || echo "::warning::Could not fully apply bench env (expected in containers)"
# Emit environment fingerprint for artifact tracking
mkdir -p target/perf
scripts/bench_env_setup.sh fingerprint | tee target/perf/bench_env_fingerprint.json || true
# Validate and report
echo "## Benchmark Environment" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
scripts/bench_env_setup.sh validate 2>&1 | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY || true
echo '```' >> $GITHUB_STEP_SUMMARY
- name: Build release binary
run: cargo build --release
- name: Check binary size
run: |
# Budget bumped 20MB -> 25MB on 2026-05-22: release profile is
# already maxed for size (opt-level=z, lto=true, codegen-units=1,
# panic=abort, strip=true). Binary genuinely grew past 22MB as more
# providers/tools landed; further shaving would require feature
# gating, which is tracked separately.
SIZE_BYTES=$(stat --printf="%s" target/release/pi)
SIZE_MB=$(echo "scale=2; $SIZE_BYTES / 1024 / 1024" | bc)
echo "## Binary Size" >> $GITHUB_STEP_SUMMARY
echo "- Size: ${SIZE_MB}MB" >> $GITHUB_STEP_SUMMARY
echo "- Budget: 25MB" >> $GITHUB_STEP_SUMMARY
if (( $(echo "$SIZE_MB > 25" | bc -l) )); then
echo "::error::Binary size ${SIZE_MB}MB exceeds 25MB budget"
exit 1
fi
echo "- Status: Within budget" >> $GITHUB_STEP_SUMMARY
- name: Run micro-benchmarks (tools)
run: cargo bench --bench tools -- --noplot
- name: Run micro-benchmarks (extensions)
run: cargo bench --bench extensions -- --noplot
- name: Run system benchmarks
run: cargo bench --bench system -- --noplot
env:
PI_BENCH_BINARY: target/release/pi
- name: Run TUI performance benchmarks (PERF-8/9)
run: cargo bench --bench tui_perf -- --noplot --save-baseline pr
- name: Generate PiJS workload perf data (JSONL)
run: |
set -euxo pipefail
PERF_PROFILE=perf
mkdir -p "target/perf/${PERF_PROFILE}"
PI_BENCH_BUILD_PROFILE="${PERF_PROFILE}" cargo run --profile "${PERF_PROFILE}" --bin pijs_workload -- --iterations 2000 --tool-calls 1 > "target/perf/${PERF_PROFILE}/pijs_workload_${PERF_PROFILE}.jsonl"
PI_BENCH_BUILD_PROFILE="${PERF_PROFILE}" cargo run --profile "${PERF_PROFILE}" --bin pijs_workload -- --iterations 2000 --tool-calls 10 >> "target/perf/${PERF_PROFILE}/pijs_workload_${PERF_PROFILE}.jsonl"
- name: Generate benchmark summary
run: |
echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Startup Time" >> $GITHUB_STEP_SUMMARY
if [ -d "target/criterion/startup" ]; then
for dir in target/criterion/startup/*/; do
name=$(basename "$dir")
if [ -f "$dir/new/estimates.json" ]; then
mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
fi
done
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Truncation" >> $GITHUB_STEP_SUMMARY
if [ -d "target/criterion/truncation" ]; then
for dir in target/criterion/truncation/*/; do
name=$(basename "$dir")
if [ -f "$dir/new/estimates.json" ]; then
mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
fi
done
fi
echo "" >> $GITHUB_STEP_SUMMARY
echo "### TUI Rendering (PERF-8)" >> $GITHUB_STEP_SUMMARY
for group in build_conversation_content view viewport_operations markdown_rendering; do
if [ -d "target/criterion/$group" ]; then
echo "**$group**" >> $GITHUB_STEP_SUMMARY
for dir in target/criterion/"$group"/*/; do
name=$(basename "$dir")
if [ -f "$dir/new/estimates.json" ]; then
mean=$(jq -r '.mean.point_estimate' "$dir/new/estimates.json" 2>/dev/null || echo "N/A")
echo "- $name: ${mean}ns" >> $GITHUB_STEP_SUMMARY
fi
done
fi
done
- name: Perf budget gate
run: cargo test --test perf_budgets -- --nocapture
- name: Restore benchmark environment
if: always()
run: sudo scripts/bench_env_setup.sh restore || true
- name: Upload benchmark results
if: always()
uses: actions/upload-artifact@v4
with:
name: benchmark-results-${{ github.sha }}
path: |
pi_agent_rust/target/criterion/
pi_agent_rust/target/perf/
retention-days: 30
# Compare with main branch baseline (only on PRs)
compare:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
needs: benchmark
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v4
with:
ref: main
path: main-branch
- uses: actions/checkout@v4
with:
path: pr-branch
- name: Install system deps (xcb) [linux]
run: |
set -euxo pipefail
sudo apt-get update
sudo apt-get install -y libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
- name: Install Rust nightly
uses: dtolnay/rust-toolchain@nightly
- name: Cache cargo registry
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
key: ${{ runner.os }}-cargo-compare-${{ hashFiles('**/Cargo.lock') }}
- name: Install critcmp
run: cargo install critcmp
- name: Build and benchmark main
working-directory: main-branch
run: |
cargo build --release
cargo bench --bench tools -- --noplot --save-baseline main 2>/dev/null || true
cargo bench --bench tui_perf -- --noplot --save-baseline main 2>/dev/null || true
- name: Copy main baseline to PR target dir
run: |
set -euxo pipefail
if [ -d "main-branch/target/criterion" ]; then
mkdir -p pr-branch/target
mkdir -p pr-branch/target/criterion
cp -R main-branch/target/criterion/. pr-branch/target/criterion/
fi
- name: Build and benchmark PR
working-directory: pr-branch
run: |
cargo build --release
cargo bench --bench tools -- --noplot --baseline main 2>/dev/null || echo "Baseline comparison skipped"
cargo bench --bench tui_perf -- --noplot --baseline main 2>/dev/null || echo "TUI baseline comparison skipped"
- name: Compare TUI performance (fail on >20% regression)
working-directory: pr-branch
env:
PERF_REGRESSION_THRESHOLD: ${{ vars.PERF_REGRESSION_THRESHOLD || '20' }}
run: |
set -euo pipefail
echo "## TUI Performance Comparison" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Run critcmp with threshold
if critcmp main pr --threshold "${PERF_REGRESSION_THRESHOLD}" 2>/dev/null; then
echo "✅ No TUI performance regressions detected (threshold: ${PERF_REGRESSION_THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
else
echo "❌ TUI performance regression detected (>${PERF_REGRESSION_THRESHOLD}%)" >> $GITHUB_STEP_SUMMARY
# Show detailed comparison
critcmp main pr 2>/dev/null >> $GITHUB_STEP_SUMMARY || true
exit 1
fi