bench(recursion): measure the verifier at real query counts, over real blocks #121
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench verifier | |
| # Manual-only (/bench-verify); separate from /bench so they never share the bench server. | |
| on: | |
| issue_comment: | |
| types: [created] | |
| concurrency: | |
| # Serialization is provided by the single self-hosted bench runner (jobs queue | |
| # for it). Never cancel a running bench: real /bench-verify comments share the | |
| # per-PR group and queue; any other comment (this workflow fires on every | |
| # issue_comment) gets a unique throwaway group so it can't sit in — or evict — | |
| # the real queue. | |
| group: ${{ startsWith(github.event.comment.body, '/bench-verify') && format('bench-verify-{0}', github.event.issue.number) || format('bench-verify-ignore-{0}', github.run_id) }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| verify: | |
| if: >- | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/bench-verify') && | |
| contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) | |
| runs-on: [self-hosted, bench] | |
| # Job cap. The verifier bench is ~5-6 min and the recursion measurement itself ~1 min, | |
| # but on a cold runner the recursion BUILDS dominate: MEASURE_CLI (release cli) once, | |
| # plus PER REF a guest build (~10-20 min) and a prover-test build for the blob dump. | |
| # All cached in /tmp for later runs, and build-std / the host cargo target are shared | |
| # across ref worktrees (see the recursion step's env) to keep the cold run under this | |
| # cap. The recursion step also has its own tighter timeout so a runaway build there | |
| # can't burn the whole job and lose the already-captured verifier result. | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Acknowledge (react + occupancy notice) | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, content: 'eyes' | |
| }); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: '⏳ **Benchmark started** on the bench server. The verifier bench takes ~5 min; the recursion-guest cycle comparison then adds guest builds — a few minutes when cached, up to ~1h on a cold run. The bench server is occupied until it finishes.' | |
| }); | |
| - name: Resolve PR head + pair count | |
| id: cfg | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUM: ${{ github.event.issue.number }} | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| run: | | |
| # Head SHA (not branch name) so fork PRs resolve and a mid-run force-push can't race. | |
| HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) | |
| echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" | |
| # Optional pair count "/bench-verify 32"; default 20, clamp [2,40]. | |
| N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-verify[[:space:]]*\([0-9]\+\).*|\1|p') | |
| N=${N:-20} | |
| if [ "$N" -lt 2 ] 2>/dev/null || [ "$N" -gt 40 ] 2>/dev/null; then | |
| echo "::warning::pair count $N out of range [2,40]; using 20" | |
| N=20 | |
| fi | |
| echo "pairs=$N" >> "$GITHUB_OUTPUT" | |
| - name: Checkout (full history for ref resolution) | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Fetch PR head commit (works for fork PRs) | |
| env: | |
| PR_NUM: ${{ github.event.issue.number }} | |
| run: git fetch origin "pull/$PR_NUM/head" --quiet | |
| - name: Add cargo to PATH | |
| run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Run verifier benchmark | |
| id: run | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| PAIRS: ${{ steps.cfg.outputs.pairs }} | |
| run: | | |
| export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" | |
| set -o pipefail | |
| scripts/bench_verify.sh "$HEAD_SHA" origin/main "$PAIRS" 2>&1 | tee /tmp/verify_out.txt | |
| sed -n '/=== Verify ABBA result/,$p' /tmp/verify_out.txt > /tmp/verify_result.txt | |
| # Additive: deterministic recursion-guest cycle+accelerator diff (PR vs main). | |
| # The measurement is one exact `execute --cycles` reading per ref (~1 min total, no | |
| # ABBA). Cold wall time is dominated by BUILDS: MEASURE_CLI (release cli) once, plus | |
| # per ref a guest build (~10-20 min) and a prover-test build for the blob dump; the | |
| # GUEST_TARGET_DIR / HOST_TARGET_DIR below share build-std and the host cargo target | |
| # across the two ref worktrees so the second ref is much cheaper (and per-worktree | |
| # disk shrinks). continue-on-error + `!cancelled()` keep this fully isolated from the | |
| # verifier bench above: a failure OR timeout here never fails the job nor clobbers the | |
| # verifier verdict, and it still runs if the verifier step failed. | |
| - name: Run recursion guest cycle benchmark | |
| id: recursion | |
| if: ${{ !cancelled() }} | |
| continue-on-error: true | |
| # Fail-fast well under the 90-min job cap so a runaway recursion build can't burn | |
| # the whole job and lose the already-captured verifier result (continue-on-error | |
| # absorbs the timeout; the job still posts the verifier verdict). Sized with | |
| # headroom over a cold shared-build run (MEASURE_CLI + 2 guest builds + 2 blob-dump | |
| # builds). | |
| timeout-minutes: 70 | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| # Share build-std and the host cargo target across ref worktrees, rooted under | |
| # the script's /tmp cache dir: cuts cold build time and per-worktree disk. | |
| GUEST_TARGET_DIR: /tmp/recursion_cycles_run/shared_guest_target | |
| HOST_TARGET_DIR: /tmp/recursion_cycles_run/shared_host_target | |
| run: | | |
| export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" | |
| set -o pipefail | |
| scripts/bench_recursion_cycles.sh "$HEAD_SHA" origin/main min 2>&1 | tee /tmp/recursion_out.txt | |
| sed -n '/=== Recursion-guest cycle/,$p' /tmp/recursion_out.txt > /tmp/recursion_result.txt | |
| - name: Post result | |
| if: always() | |
| uses: actions/github-script@v7 | |
| env: | |
| HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} | |
| PAIRS: ${{ steps.cfg.outputs.pairs }} | |
| OUTCOME: ${{ steps.run.outcome }} | |
| RECURSION_OUTCOME: ${{ steps.recursion.outcome }} | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } }; | |
| // Bound any raw-log fallback so a future header rename (empty *_result.txt) | |
| // can't dump the entire build log into the PR comment. | |
| const tail = (s, n) => s.split('\n').slice(-n).join('\n'); | |
| const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS; | |
| let body = `## Verifier benchmark — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`; | |
| if (process.env.OUTCOME === 'success') { | |
| const res = read('/tmp/verify_result.txt') || tail(read('/tmp/verify_out.txt'), 30); | |
| body += res + '\n'; | |
| body += '\n<sub>Drift-free interleaved A/B/B/A measurement. - = PR faster. '; | |
| body += 'Trust the verdict when paired-t and Wilcoxon agree.</sub>\n'; | |
| } else { | |
| body += `❌ Run failed. Last log lines:\n\n` + '```\n' + tail(read('/tmp/verify_out.txt'), 30) + '\n```\n'; | |
| } | |
| // Additive recursion-guest cycle section, kept clearly separated from the | |
| // verifier verdict above so a failure here can't change how the bench reads. | |
| body += '\n---\n\n## Recursion guest cycles (main vs PR)\n\n'; | |
| if (process.env.RECURSION_OUTCOME === 'success') { | |
| const rec = read('/tmp/recursion_result.txt') || tail(read('/tmp/recursion_out.txt'), 20); | |
| if (rec) { | |
| body += rec + '\n'; | |
| } else { | |
| body += '_No recursion comparison output was captured._\n'; | |
| } | |
| } else { | |
| const rtail = tail(read('/tmp/recursion_out.txt'), 20); | |
| body += '⚠️ Recursion cycle bench did not complete (does not affect the verifier verdict above).'; | |
| body += rtail ? ' Last log lines:\n\n' + '```\n' + rtail + '\n```\n' : '\n'; | |
| } | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const existing = comments.find(c => | |
| c.user.type === 'Bot' && c.body.includes('Verifier benchmark —')); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| comment_id: existing.id, body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, body | |
| }); | |
| } |