bench(recursion): measure the verifier at real query counts, over real blocks #1457
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark (PR) | |
| on: | |
| workflow_dispatch: | |
| issue_comment: | |
| types: [created] | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'prover/**' | |
| - 'crypto/**' | |
| - 'executor/**' | |
| - 'bin/cli/**' | |
| - 'tooling/ethrex-fixtures/**' | |
| # Uncomment to auto-run on PRs: | |
| # pull_request: | |
| # branches: [main] | |
| # paths: | |
| # - 'prover/**' | |
| # - 'crypto/**' | |
| # - 'executor/**' | |
| # - 'bin/cli/**' | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| actions: read | |
| concurrency: | |
| # Runner serializes; never cancel a running bench (group is already unique per run for comment/push events). | |
| group: benchmark-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: false | |
| env: | |
| # Headline program: the ethrex guest ELF proven against a 20-transfer block | |
| # (distinct sender -> distinct recipient per tx). One ELF; the workload is the | |
| # private input (rkyv ProgramInput), generated in-job and gitignored (see the | |
| # "Generate ethrex bench fixtures" step). | |
| ELF: executor/program_artifacts/rust/ethrex.elf | |
| INPUT: executor/tests/ethrex_bench_20.bin | |
| BENCH_RUNS_PR: 3 | |
| # Cheap-tier screen: catches regressions down to ~1.5% on its own and leaves | |
| # smaller/ambiguous deltas to the manual drift-free ABBA tiebreaker. Pushing either | |
| # side past 5 buys little here (the cached comparison can't beat the ~1% drift wall), | |
| # so the per-PR run count is also capped at 5 (clamp below). | |
| BENCH_RUNS_BASELINE: 5 | |
| # Memory-scaling sweep: same ELF, different N-transfer inputs. GROWTH_PROGRAMS | |
| # are the generated (gitignored) fixture basenames in executor/tests/; GROWTH_STEPS | |
| # the matching transfer counts (x-axis; slope is MB per transfer). | |
| GROWTH_PROGRAMS: "ethrex_bench_4 ethrex_bench_8 ethrex_bench_12 ethrex_bench_16 ethrex_bench_20" | |
| GROWTH_STEPS: "4 8 12 16 20" | |
| jobs: | |
| benchmark: | |
| runs-on: [self-hosted, bench] | |
| # Skip unless: push to main, workflow_dispatch, or "/bench" comment on a PR | |
| if: >- | |
| github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| startsWith(github.event.comment.body, '/bench') && | |
| !startsWith(github.event.comment.body, '/bench-abba') && | |
| !startsWith(github.event.comment.body, '/bench-gpu') && | |
| !startsWith(github.event.comment.body, '/bench-verify') && | |
| contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)) | |
| steps: | |
| - name: React to comment | |
| if: github.event_name == 'issue_comment' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.reactions.createForIssueComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: context.payload.comment.id, | |
| content: 'eyes' | |
| }); | |
| - name: Get PR head ref | |
| id: pr-ref | |
| if: github.event_name == 'issue_comment' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUM: ${{ github.event.issue.number }} | |
| run: | | |
| SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) | |
| echo "sha=$SHA" >> "$GITHUB_OUTPUT" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.pr-ref.outputs.sha || github.sha }} | |
| - name: Add cargo to PATH | |
| run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - name: Build ethrex guest ELF | |
| run: | | |
| # Self-provision the RV64 sysroot in a user-writable dir (matches the | |
| # nightly bench job); make picks it up via SYSROOT_DIR ?= and passes it | |
| # to clang as --sysroot. The ELF is gitignored and persists across the | |
| # baseline `git checkout`, so the same workload is proven on both sides. | |
| export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" | |
| make executor/program_artifacts/rust/ethrex.elf | |
| - name: Generate ethrex bench fixtures | |
| run: | | |
| # Generated, not committed (gitignored via executor/.gitignore). They are | |
| # untracked, so they survive the baseline `git checkout origin/main` below — | |
| # the SAME workload (ELF + inputs) is proven on both the PR and main sides. | |
| # distinct = N independent genesis-funded senders -> N distinct recipients. | |
| ( cd tooling/ethrex-fixtures && cargo build --release ) | |
| GEN=tooling/ethrex-fixtures/target/release/ethrex-fixtures | |
| for n in $GROWTH_STEPS; do | |
| "$GEN" "$n" "executor/tests/ethrex_bench_${n}.bin" distinct | |
| done | |
| - name: Build CLI (PR) | |
| run: cargo build --release -p cli --features jemalloc-stats | |
| - name: Determine run count | |
| id: config | |
| env: | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: | | |
| # Growth benchmarks: run on /bench-growth, push to main, or workflow_dispatch | |
| # Skip on plain /bench to keep it fast | |
| if [ "$EVENT_NAME" = "issue_comment" ] && echo "$COMMENT_BODY" | grep -q '^/bench-growth'; then | |
| echo "run_growth=true" >> "$GITHUB_OUTPUT" | |
| elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| echo "run_growth=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run_growth=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "$EVENT_NAME" = "issue_comment" ]; then | |
| CUSTOM_N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench[[:space:]]*\([0-9]\+\).*|\1|p') | |
| RUNS=${CUSTOM_N:-$BENCH_RUNS_PR} | |
| elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "workflow_dispatch" ]; then | |
| RUNS=$BENCH_RUNS_BASELINE | |
| else | |
| RUNS=$BENCH_RUNS_PR | |
| fi | |
| # Clamp to 1-5. Beyond 5 the single-session cached comparison barely improves | |
| # (it can't beat the ~1% drift wall); use the ABBA tiebreaker for finer deltas. | |
| if [ "$RUNS" -lt 1 ] 2>/dev/null || [ "$RUNS" -gt 5 ] 2>/dev/null; then | |
| echo "::warning::Run count $RUNS out of range [1,5], defaulting to $BENCH_RUNS_PR" | |
| RUNS=$BENCH_RUNS_PR | |
| fi | |
| echo "runs=$RUNS" >> "$GITHUB_OUTPUT" | |
| # Optional table parallelism for the HEADLINE benchmark only (the memory | |
| # growth sweep always runs at default parallelism). `/bench k=N` overrides; | |
| # otherwise default (cores/3). /bench-growth no longer forces k=1. | |
| TABLE_K="" | |
| if [ "$EVENT_NAME" = "issue_comment" ]; then | |
| TABLE_K=$(echo "$COMMENT_BODY" | grep -o 'k=[0-9]*' | head -1 | cut -d= -f2) | |
| fi | |
| echo "table_parallelism=${TABLE_K:-}" >> "$GITHUB_OUTPUT" | |
| echo "k_tag=${TABLE_K:-auto}" >> "$GITHUB_OUTPUT" | |
| if [ -n "$TABLE_K" ]; then | |
| echo "Using $RUNS iterations, TABLE_PARALLELISM=$TABLE_K" | |
| else | |
| echo "Using $RUNS iterations, TABLE_PARALLELISM=default" | |
| fi | |
| - name: Benchmark PR | |
| id: pr | |
| env: | |
| RUNS: ${{ steps.config.outputs.runs }} | |
| TABLE_PARALLELISM: ${{ steps.config.outputs.table_parallelism }} | |
| run: | | |
| if [ -n "$TABLE_PARALLELISM" ]; then | |
| export TABLE_PARALLELISM | |
| echo "TABLE_PARALLELISM=$TABLE_PARALLELISM" | |
| fi | |
| TIMES="" | |
| HEAPS="" | |
| for i in $(seq 1 $RUNS); do | |
| echo "--- Run $i/$RUNS ---" | |
| ./target/release/cli prove "$ELF" --private-input "$INPUT" -o /tmp/proof.bin --time \ | |
| | tee /tmp/cli_output_$i.txt | |
| rm -f /tmp/proof.bin | |
| T=$(grep -o 'Proving time: [0-9.]*' /tmp/cli_output_$i.txt | awk '{print $3}') | |
| H=$(grep -o 'Peak heap: [0-9]*' /tmp/cli_output_$i.txt | awk '{print $3}') | |
| if [ -z "$T" ] || [ -z "$H" ]; then | |
| echo "::error::Failed to parse metrics from run $i" | |
| cat /tmp/cli_output_$i.txt | |
| exit 1 | |
| fi | |
| TIMES="$TIMES $T" | |
| HEAPS="$HEAPS $H" | |
| done | |
| # Median position (works for odd N; for even N picks lower-middle) | |
| MEDIAN_POS=$(( (RUNS + 1) / 2 )) | |
| TIME_MEDIAN=$(echo $TIMES | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") | |
| HEAP_MEDIAN=$(echo $HEAPS | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") | |
| if [ -z "$HEAP_MEDIAN" ] || [ -z "$TIME_MEDIAN" ]; then | |
| echo "::error::Failed to compute median metrics" | |
| exit 1 | |
| fi | |
| # Spread: (max - min) / median * 100 | |
| TIME_MIN=$(echo $TIMES | tr ' ' '\n' | sort -n | head -1) | |
| TIME_MAX=$(echo $TIMES | tr ' ' '\n' | sort -n | tail -1) | |
| TIME_SPREAD=$(awk "BEGIN { if ($TIME_MEDIAN > 0) printf \"%.1f\", (($TIME_MAX - $TIME_MIN) / $TIME_MEDIAN) * 100; else print \"0.0\" }") | |
| HEAP_MIN=$(echo $HEAPS | tr ' ' '\n' | sort -n | head -1) | |
| HEAP_MAX=$(echo $HEAPS | tr ' ' '\n' | sort -n | tail -1) | |
| HEAP_SPREAD=$(awk "BEGIN { if ($HEAP_MEDIAN > 0) printf \"%.1f\", (($HEAP_MAX - $HEAP_MIN) / $HEAP_MEDIAN) * 100; else print \"0.0\" }") | |
| ALL_TIMES=$(echo $TIMES | tr ' ' '\n' | paste -sd '/' -) | |
| ALL_HEAPS=$(echo $HEAPS | tr ' ' '\n' | paste -sd '/' -) | |
| echo "peak_mb=$HEAP_MEDIAN" >> "$GITHUB_OUTPUT" | |
| echo "time_s=$TIME_MEDIAN" >> "$GITHUB_OUTPUT" | |
| echo "time_spread=$TIME_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "heap_spread=$HEAP_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "all_times=$ALL_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "all_heaps=$ALL_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "runs=$RUNS" >> "$GITHUB_OUTPUT" | |
| echo "peak_mb=$HEAP_MEDIAN" > /tmp/metrics.txt | |
| echo "time_s=$TIME_MEDIAN" >> /tmp/metrics.txt | |
| echo "time_spread=$TIME_SPREAD" >> /tmp/metrics.txt | |
| echo "heap_spread=$HEAP_SPREAD" >> /tmp/metrics.txt | |
| echo "all_times=$ALL_TIMES" >> /tmp/metrics.txt | |
| echo "all_heaps=$ALL_HEAPS" >> /tmp/metrics.txt | |
| echo "runs=$RUNS" >> /tmp/metrics.txt | |
| - name: Memory growth (PR) | |
| id: pr-growth | |
| if: steps.config.outputs.run_growth == 'true' | |
| run: | | |
| PROGRAMS=($GROWTH_PROGRAMS) | |
| STEPS_ARR=($GROWTH_STEPS) | |
| GROWTH_HEAPS="" | |
| GROWTH_TIMES="" | |
| # 1 sample/point: run-to-run heap is ~deterministic (<0.3%), so an extra | |
| # transfer-count point buys more slope accuracy than a replicate. | |
| SAMPLES=1 | |
| for idx in "${!PROGRAMS[@]}"; do | |
| prog="${PROGRAMS[$idx]}" | |
| INPUT_PATH="executor/tests/${prog}.bin" | |
| SAMPLE_HEAPS="" | |
| SAMPLE_TIMES="" | |
| for s in $(seq 1 $SAMPLES); do | |
| echo "--- Growth: $prog (sample $s/$SAMPLES, default parallelism) ---" | |
| ./target/release/cli prove "$ELF" --private-input "$INPUT_PATH" -o /tmp/proof.bin --time \ | |
| | tee /tmp/growth_${prog}_${s}.txt | |
| rm -f /tmp/proof.bin | |
| T=$(grep -o 'Proving time: [0-9.]*' /tmp/growth_${prog}_${s}.txt | awk '{print $3}') | |
| H=$(grep -o 'Peak heap: [0-9]*' /tmp/growth_${prog}_${s}.txt | awk '{print $3}') | |
| if [ -z "$T" ] || [ -z "$H" ]; then | |
| echo "::error::Failed to parse growth metrics for $prog sample $s" | |
| exit 1 | |
| fi | |
| SAMPLE_HEAPS="$SAMPLE_HEAPS $H" | |
| SAMPLE_TIMES="$SAMPLE_TIMES $T" | |
| done | |
| H=$(echo $SAMPLE_HEAPS | tr ' ' '\n' | sort -n | head -1) | |
| T=$(echo $SAMPLE_TIMES | tr ' ' '\n' | sort -n | head -1) | |
| GROWTH_HEAPS="${GROWTH_HEAPS:+$GROWTH_HEAPS/}$H" | |
| GROWTH_TIMES="${GROWTH_TIMES:+$GROWTH_TIMES/}$T" | |
| done | |
| # Linear regression: heap (MB) vs transfer count (slope = MB per transfer) | |
| STEPS_SLASH=$(echo "${STEPS_ARR[@]}" | tr ' ' '/') | |
| read SLOPE R2 <<< $(awk -v steps="$STEPS_SLASH" -v heaps="$GROWTH_HEAPS" 'BEGIN { | |
| n = split(steps, xs, "/") | |
| split(heaps, ys, "/") | |
| sx = 0; sy = 0; sxy = 0; sx2 = 0 | |
| for (i = 1; i <= n; i++) { | |
| x = xs[i]; y = ys[i] + 0 | |
| sx += x; sy += y; sxy += x * y; sx2 += x * x | |
| } | |
| d = n * sx2 - sx * sx | |
| if (d == 0) { print "0 0.0000"; exit } | |
| slope = (n * sxy - sx * sy) / d | |
| my = sy / n; ss_tot = 0; ss_res = 0 | |
| for (i = 1; i <= n; i++) { | |
| x = xs[i]; y = ys[i] + 0 | |
| pred = slope * x + (sy - slope * sx) / n | |
| ss_res += (y - pred) * (y - pred) | |
| ss_tot += (y - my) * (y - my) | |
| } | |
| r2 = (ss_tot > 0) ? 1 - ss_res / ss_tot : 0 | |
| printf "%.0f %.4f\n", slope, r2 | |
| }') | |
| echo "growth_steps=$STEPS_SLASH" >> "$GITHUB_OUTPUT" | |
| echo "growth_heaps=$GROWTH_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "growth_times=$GROWTH_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "growth_slope_mb=$SLOPE" >> "$GITHUB_OUTPUT" | |
| echo "growth_r2=$R2" >> "$GITHUB_OUTPUT" | |
| # Append to metrics artifact | |
| echo "growth_steps=$STEPS_SLASH" >> /tmp/metrics.txt | |
| echo "growth_heaps=$GROWTH_HEAPS" >> /tmp/metrics.txt | |
| echo "growth_times=$GROWTH_TIMES" >> /tmp/metrics.txt | |
| echo "growth_slope_mb=$SLOPE" >> /tmp/metrics.txt | |
| echo "growth_r2=$R2" >> /tmp/metrics.txt | |
| - name: Upload metrics artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-metrics-${{ steps.pr-ref.outputs.sha || github.sha }}-k${{ steps.config.outputs.k_tag }} | |
| path: /tmp/metrics.txt | |
| retention-days: 90 | |
| # --- Baseline: try artifact first, fall back to building main --- | |
| - name: Download baseline artifact | |
| id: baseline-artifact | |
| if: github.event_name != 'push' && github.event_name != 'workflow_dispatch' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| K_TAG: ${{ steps.config.outputs.k_tag }} | |
| run: | | |
| RUN_ID=$(gh run list \ | |
| -w benchmark-pr.yml -b main -s completed \ | |
| --json databaseId,conclusion,event \ | |
| -q '[.[] | select(.conclusion=="success" and (.event=="push" or .event=="workflow_dispatch"))][0].databaseId') | |
| if [ -n "$RUN_ID" ]; then | |
| if gh run download "$RUN_ID" -D baseline/ -p "benchmark-metrics-*-k${K_TAG}"; then | |
| BASELINE_FILE=$(ls -t baseline/*/metrics.txt 2>/dev/null | head -1) | |
| if [ -n "$BASELINE_FILE" ]; then | |
| echo "found=true" >> "$GITHUB_OUTPUT" | |
| echo "peak_mb=$(grep 'peak_mb=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "time_s=$(grep 'time_s=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "time_spread=$(grep 'time_spread=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "heap_spread=$(grep 'heap_spread=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "all_times=$(grep 'all_times=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "all_heaps=$(grep 'all_heaps=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "runs=$(grep 'runs=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "growth_heaps=$(grep 'growth_heaps=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "growth_times=$(grep 'growth_times=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "growth_slope_mb=$(grep 'growth_slope_mb=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "growth_r2=$(grep 'growth_r2=' "$BASELINE_FILE" | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| fi | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| - name: Build and benchmark main (no cached baseline) | |
| id: baseline-run | |
| if: github.event_name != 'push' && github.event_name != 'workflow_dispatch' && steps.baseline-artifact.outputs.found != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUNS: ${{ steps.config.outputs.runs }} | |
| RUN_GROWTH: ${{ steps.config.outputs.run_growth }} | |
| TABLE_PARALLELISM: ${{ steps.config.outputs.table_parallelism }} | |
| run: | | |
| if [ -n "$TABLE_PARALLELISM" ]; then | |
| export TABLE_PARALLELISM | |
| echo "TABLE_PARALLELISM=$TABLE_PARALLELISM" | |
| fi | |
| # Save current HEAD | |
| PR_SHA=$(git rev-parse HEAD) | |
| # Checkout main and rebuild the prover (CLI) only. The workload — the gitignored | |
| # ethrex ELF and the generated, untracked bench fixtures — is left untouched by | |
| # the checkout, so the same inputs are proven on both the PR and main sides. | |
| git fetch origin main | |
| git checkout origin/main | |
| cargo build --release -p cli --features jemalloc-stats | |
| # --- Primary benchmark (ethrex 20 transfers) --- | |
| TIMES="" | |
| HEAPS="" | |
| for i in $(seq 1 $RUNS); do | |
| echo "--- Baseline run $i/$RUNS ---" | |
| ./target/release/cli prove "$ELF" --private-input "$INPUT" -o /tmp/proof.bin --time \ | |
| | tee /tmp/baseline_output_$i.txt | |
| rm -f /tmp/proof.bin | |
| T=$(grep -o 'Proving time: [0-9.]*' /tmp/baseline_output_$i.txt | awk '{print $3}') | |
| H=$(grep -o 'Peak heap: [0-9]*' /tmp/baseline_output_$i.txt | awk '{print $3}') | |
| if [ -z "$T" ] || [ -z "$H" ]; then | |
| echo "::error::Failed to parse baseline metrics from run $i" | |
| cat /tmp/baseline_output_$i.txt | |
| exit 1 | |
| fi | |
| TIMES="$TIMES $T" | |
| HEAPS="$HEAPS $H" | |
| done | |
| MEDIAN_POS=$(( (RUNS + 1) / 2 )) | |
| TIME_MEDIAN=$(echo $TIMES | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") | |
| HEAP_MEDIAN=$(echo $HEAPS | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") | |
| if [ -z "$HEAP_MEDIAN" ] || [ -z "$TIME_MEDIAN" ]; then | |
| echo "::error::Failed to compute baseline median metrics" | |
| exit 1 | |
| fi | |
| TIME_MIN=$(echo $TIMES | tr ' ' '\n' | sort -n | head -1) | |
| TIME_MAX=$(echo $TIMES | tr ' ' '\n' | sort -n | tail -1) | |
| TIME_SPREAD=$(awk "BEGIN { if ($TIME_MEDIAN > 0) printf \"%.1f\", (($TIME_MAX - $TIME_MIN) / $TIME_MEDIAN) * 100; else print \"0.0\" }") | |
| HEAP_MIN=$(echo $HEAPS | tr ' ' '\n' | sort -n | head -1) | |
| HEAP_MAX=$(echo $HEAPS | tr ' ' '\n' | sort -n | tail -1) | |
| HEAP_SPREAD=$(awk "BEGIN { if ($HEAP_MEDIAN > 0) printf \"%.1f\", (($HEAP_MAX - $HEAP_MIN) / $HEAP_MEDIAN) * 100; else print \"0.0\" }") | |
| ALL_TIMES=$(echo $TIMES | tr ' ' '\n' | paste -sd '/' -) | |
| ALL_HEAPS=$(echo $HEAPS | tr ' ' '\n' | paste -sd '/' -) | |
| echo "peak_mb=$HEAP_MEDIAN" >> "$GITHUB_OUTPUT" | |
| echo "time_s=$TIME_MEDIAN" >> "$GITHUB_OUTPUT" | |
| echo "time_spread=$TIME_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "heap_spread=$HEAP_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "all_times=$ALL_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "all_heaps=$ALL_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "runs=$RUNS" >> "$GITHUB_OUTPUT" | |
| # --- Growth benchmarks (default parallelism, 1 sample each) --- | |
| # Only run if /bench-growth, push, or workflow_dispatch | |
| if [ "$RUN_GROWTH" != "true" ]; then | |
| echo "Skipping growth benchmarks (use /bench-growth to enable)" | |
| else | |
| PROGRAMS=($GROWTH_PROGRAMS) | |
| STEPS_ARR=($GROWTH_STEPS) | |
| GROWTH_HEAPS="" | |
| GROWTH_TIMES="" | |
| SAMPLES=1 | |
| for idx in "${!PROGRAMS[@]}"; do | |
| prog="${PROGRAMS[$idx]}" | |
| INPUT_PATH="executor/tests/${prog}.bin" | |
| SAMPLE_HEAPS="" | |
| SAMPLE_TIMES="" | |
| for s in $(seq 1 $SAMPLES); do | |
| echo "--- Baseline growth: $prog (sample $s/$SAMPLES, default parallelism) ---" | |
| ./target/release/cli prove "$ELF" --private-input "$INPUT_PATH" -o /tmp/proof.bin --time \ | |
| | tee /tmp/baseline_growth_${prog}_${s}.txt | |
| rm -f /tmp/proof.bin | |
| T=$(grep -o 'Proving time: [0-9.]*' /tmp/baseline_growth_${prog}_${s}.txt | awk '{print $3}') | |
| H=$(grep -o 'Peak heap: [0-9]*' /tmp/baseline_growth_${prog}_${s}.txt | awk '{print $3}') | |
| if [ -z "$T" ] || [ -z "$H" ]; then | |
| echo "::error::Failed to parse baseline growth metrics for $prog sample $s" | |
| exit 1 | |
| fi | |
| SAMPLE_HEAPS="$SAMPLE_HEAPS $H" | |
| SAMPLE_TIMES="$SAMPLE_TIMES $T" | |
| done | |
| H=$(echo $SAMPLE_HEAPS | tr ' ' '\n' | sort -n | head -1) | |
| T=$(echo $SAMPLE_TIMES | tr ' ' '\n' | sort -n | head -1) | |
| GROWTH_HEAPS="${GROWTH_HEAPS:+$GROWTH_HEAPS/}$H" | |
| GROWTH_TIMES="${GROWTH_TIMES:+$GROWTH_TIMES/}$T" | |
| done | |
| STEPS_SLASH=$(echo "${STEPS_ARR[@]}" | tr ' ' '/') | |
| read SLOPE R2 <<< $(awk -v steps="$STEPS_SLASH" -v heaps="$GROWTH_HEAPS" 'BEGIN { | |
| n = split(steps, xs, "/") | |
| split(heaps, ys, "/") | |
| sx = 0; sy = 0; sxy = 0; sx2 = 0 | |
| for (i = 1; i <= n; i++) { | |
| x = xs[i]; y = ys[i] + 0 | |
| sx += x; sy += y; sxy += x * y; sx2 += x * x | |
| } | |
| d = n * sx2 - sx * sx | |
| if (d == 0) { print "0 0.0000"; exit } | |
| slope = (n * sxy - sx * sy) / d | |
| my = sy / n; ss_tot = 0; ss_res = 0 | |
| for (i = 1; i <= n; i++) { | |
| x = xs[i]; y = ys[i] + 0 | |
| pred = slope * x + (sy - slope * sx) / n | |
| ss_res += (y - pred) * (y - pred) | |
| ss_tot += (y - my) * (y - my) | |
| } | |
| r2 = (ss_tot > 0) ? 1 - ss_res / ss_tot : 0 | |
| printf "%.0f %.4f\n", slope, r2 | |
| }') | |
| echo "growth_heaps=$GROWTH_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "growth_times=$GROWTH_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "growth_slope_mb=$SLOPE" >> "$GITHUB_OUTPUT" | |
| echo "growth_r2=$R2" >> "$GITHUB_OUTPUT" | |
| fi # end run_growth check | |
| # Restore PR checkout | |
| git checkout "$PR_SHA" | |
| # --- Compare --- | |
| - name: Compare | |
| id: compare | |
| if: github.event_name != 'push' && github.event_name != 'workflow_dispatch' | |
| env: | |
| # Baseline artifact outputs | |
| BASELINE_FOUND: ${{ steps.baseline-artifact.outputs.found }} | |
| BA_PEAK_MB: ${{ steps.baseline-artifact.outputs.peak_mb }} | |
| BA_TIME_S: ${{ steps.baseline-artifact.outputs.time_s }} | |
| BA_TIME_SPREAD: ${{ steps.baseline-artifact.outputs.time_spread }} | |
| BA_HEAP_SPREAD: ${{ steps.baseline-artifact.outputs.heap_spread }} | |
| BA_ALL_TIMES: ${{ steps.baseline-artifact.outputs.all_times }} | |
| BA_ALL_HEAPS: ${{ steps.baseline-artifact.outputs.all_heaps }} | |
| BA_RUNS: ${{ steps.baseline-artifact.outputs.runs }} | |
| BA_GROWTH_HEAPS: ${{ steps.baseline-artifact.outputs.growth_heaps }} | |
| BA_GROWTH_TIMES: ${{ steps.baseline-artifact.outputs.growth_times }} | |
| BA_GROWTH_SLOPE: ${{ steps.baseline-artifact.outputs.growth_slope_mb }} | |
| BA_GROWTH_R2: ${{ steps.baseline-artifact.outputs.growth_r2 }} | |
| # Baseline run outputs | |
| BR_PEAK_MB: ${{ steps.baseline-run.outputs.peak_mb }} | |
| BR_TIME_S: ${{ steps.baseline-run.outputs.time_s }} | |
| BR_TIME_SPREAD: ${{ steps.baseline-run.outputs.time_spread }} | |
| BR_HEAP_SPREAD: ${{ steps.baseline-run.outputs.heap_spread }} | |
| BR_ALL_TIMES: ${{ steps.baseline-run.outputs.all_times }} | |
| BR_ALL_HEAPS: ${{ steps.baseline-run.outputs.all_heaps }} | |
| BR_RUNS: ${{ steps.baseline-run.outputs.runs }} | |
| BR_GROWTH_HEAPS: ${{ steps.baseline-run.outputs.growth_heaps }} | |
| BR_GROWTH_TIMES: ${{ steps.baseline-run.outputs.growth_times }} | |
| BR_GROWTH_SLOPE: ${{ steps.baseline-run.outputs.growth_slope_mb }} | |
| BR_GROWTH_R2: ${{ steps.baseline-run.outputs.growth_r2 }} | |
| # PR outputs | |
| CURRENT_PEAK: ${{ steps.pr.outputs.peak_mb }} | |
| CURRENT_TIME: ${{ steps.pr.outputs.time_s }} | |
| PR_TIME_SPREAD: ${{ steps.pr.outputs.time_spread }} | |
| PR_HEAP_SPREAD: ${{ steps.pr.outputs.heap_spread }} | |
| PR_ALL_TIMES: ${{ steps.pr.outputs.all_times }} | |
| PR_ALL_HEAPS: ${{ steps.pr.outputs.all_heaps }} | |
| PR_RUNS: ${{ steps.pr.outputs.runs }} | |
| # PR growth outputs | |
| PR_GROWTH_HEAPS: ${{ steps.pr-growth.outputs.growth_heaps }} | |
| PR_GROWTH_TIMES: ${{ steps.pr-growth.outputs.growth_times }} | |
| PR_GROWTH_SLOPE: ${{ steps.pr-growth.outputs.growth_slope_mb }} | |
| PR_GROWTH_R2: ${{ steps.pr-growth.outputs.growth_r2 }} | |
| run: | | |
| # Pick baseline source | |
| if [ "$BASELINE_FOUND" = "true" ]; then | |
| BASELINE_PEAK="$BA_PEAK_MB" | |
| BASELINE_TIME="$BA_TIME_S" | |
| BASELINE_SRC="cached" | |
| BASELINE_TIME_SPREAD="$BA_TIME_SPREAD" | |
| BASELINE_HEAP_SPREAD="$BA_HEAP_SPREAD" | |
| BASELINE_ALL_TIMES="$BA_ALL_TIMES" | |
| BASELINE_ALL_HEAPS="$BA_ALL_HEAPS" | |
| BASELINE_RUNS="$BA_RUNS" | |
| BASELINE_GROWTH_HEAPS="$BA_GROWTH_HEAPS" | |
| BASELINE_GROWTH_TIMES="$BA_GROWTH_TIMES" | |
| BASELINE_GROWTH_SLOPE="$BA_GROWTH_SLOPE" | |
| BASELINE_GROWTH_R2="$BA_GROWTH_R2" | |
| else | |
| BASELINE_PEAK="$BR_PEAK_MB" | |
| BASELINE_TIME="$BR_TIME_S" | |
| BASELINE_SRC="built from main" | |
| BASELINE_TIME_SPREAD="$BR_TIME_SPREAD" | |
| BASELINE_HEAP_SPREAD="$BR_HEAP_SPREAD" | |
| BASELINE_ALL_TIMES="$BR_ALL_TIMES" | |
| BASELINE_ALL_HEAPS="$BR_ALL_HEAPS" | |
| BASELINE_RUNS="$BR_RUNS" | |
| BASELINE_GROWTH_HEAPS="$BR_GROWTH_HEAPS" | |
| BASELINE_GROWTH_TIMES="$BR_GROWTH_TIMES" | |
| BASELINE_GROWTH_SLOPE="$BR_GROWTH_SLOPE" | |
| BASELINE_GROWTH_R2="$BR_GROWTH_R2" | |
| fi | |
| if [ -z "$BASELINE_PEAK" ] || [ "$BASELINE_PEAK" -eq 0 ] 2>/dev/null || | |
| [ -z "$BASELINE_TIME" ]; then | |
| echo "::error::Invalid baseline values: peak=$BASELINE_PEAK time=$BASELINE_TIME" | |
| exit 1 | |
| fi | |
| PEAK_DIFF=$((CURRENT_PEAK - BASELINE_PEAK)) | |
| PEAK_PCT=$(awk "BEGIN { printf \"%.1f\", ($PEAK_DIFF * 100) / $BASELINE_PEAK }") | |
| TIME_DIFF=$(awk "BEGIN { printf \"%.3f\", $CURRENT_TIME - $BASELINE_TIME }") | |
| TIME_PCT=$(awk "BEGIN { printf \"%.1f\", (($CURRENT_TIME - $BASELINE_TIME) * 100) / $BASELINE_TIME }") | |
| echo "baseline_peak=$BASELINE_PEAK" >> "$GITHUB_OUTPUT" | |
| echo "baseline_time=$BASELINE_TIME" >> "$GITHUB_OUTPUT" | |
| echo "baseline_src=$BASELINE_SRC" >> "$GITHUB_OUTPUT" | |
| echo "peak_diff=$PEAK_DIFF" >> "$GITHUB_OUTPUT" | |
| echo "peak_pct=$PEAK_PCT" >> "$GITHUB_OUTPUT" | |
| echo "time_diff=$TIME_DIFF" >> "$GITHUB_OUTPUT" | |
| echo "time_pct=$TIME_PCT" >> "$GITHUB_OUTPUT" | |
| echo "pr_time_spread=$PR_TIME_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "pr_heap_spread=$PR_HEAP_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "pr_all_times=$PR_ALL_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "pr_all_heaps=$PR_ALL_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "pr_runs=$PR_RUNS" >> "$GITHUB_OUTPUT" | |
| echo "baseline_time_spread=$BASELINE_TIME_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "baseline_heap_spread=$BASELINE_HEAP_SPREAD" >> "$GITHUB_OUTPUT" | |
| echo "baseline_all_times=$BASELINE_ALL_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "baseline_all_heaps=$BASELINE_ALL_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "baseline_runs=$BASELINE_RUNS" >> "$GITHUB_OUTPUT" | |
| # Growth comparison | |
| echo "pr_growth_heaps=$PR_GROWTH_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "pr_growth_times=$PR_GROWTH_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "pr_growth_slope=$PR_GROWTH_SLOPE" >> "$GITHUB_OUTPUT" | |
| echo "pr_growth_r2=$PR_GROWTH_R2" >> "$GITHUB_OUTPUT" | |
| echo "baseline_growth_heaps=$BASELINE_GROWTH_HEAPS" >> "$GITHUB_OUTPUT" | |
| echo "baseline_growth_times=$BASELINE_GROWTH_TIMES" >> "$GITHUB_OUTPUT" | |
| echo "baseline_growth_slope=$BASELINE_GROWTH_SLOPE" >> "$GITHUB_OUTPUT" | |
| echo "baseline_growth_r2=$BASELINE_GROWTH_R2" >> "$GITHUB_OUTPUT" | |
| # Growth slope comparison | |
| if [ -n "$BASELINE_GROWTH_SLOPE" ] && [ -n "$PR_GROWTH_SLOPE" ]; then | |
| SLOPE_DIFF=$((PR_GROWTH_SLOPE - BASELINE_GROWTH_SLOPE)) | |
| SLOPE_PCT=$(awk "BEGIN { if ($BASELINE_GROWTH_SLOPE != 0) printf \"%.1f\", ($SLOPE_DIFF * 100) / $BASELINE_GROWTH_SLOPE; else print \"0.0\" }") | |
| echo "growth_slope_diff=$SLOPE_DIFF" >> "$GITHUB_OUTPUT" | |
| echo "growth_slope_pct=$SLOPE_PCT" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Comment on PR | |
| if: github.event_name != 'push' && github.event_name != 'workflow_dispatch' | |
| uses: actions/github-script@v7 | |
| env: | |
| PR_PEAK: ${{ steps.pr.outputs.peak_mb }} | |
| PR_TIME: ${{ steps.pr.outputs.time_s }} | |
| BASELINE_PEAK: ${{ steps.compare.outputs.baseline_peak }} | |
| BASELINE_TIME: ${{ steps.compare.outputs.baseline_time }} | |
| BASELINE_SRC: ${{ steps.compare.outputs.baseline_src }} | |
| PEAK_PCT: ${{ steps.compare.outputs.peak_pct }} | |
| TIME_PCT: ${{ steps.compare.outputs.time_pct }} | |
| PEAK_DIFF: ${{ steps.compare.outputs.peak_diff }} | |
| TIME_DIFF: ${{ steps.compare.outputs.time_diff }} | |
| PR_RUNS: ${{ steps.compare.outputs.pr_runs }} | |
| PR_TIME_SPREAD: ${{ steps.compare.outputs.pr_time_spread }} | |
| PR_HEAP_SPREAD: ${{ steps.compare.outputs.pr_heap_spread }} | |
| PR_ALL_TIMES: ${{ steps.compare.outputs.pr_all_times }} | |
| PR_ALL_HEAPS: ${{ steps.compare.outputs.pr_all_heaps }} | |
| BASE_TIME_SPREAD: ${{ steps.compare.outputs.baseline_time_spread }} | |
| BASE_HEAP_SPREAD: ${{ steps.compare.outputs.baseline_heap_spread }} | |
| BASE_ALL_TIMES: ${{ steps.compare.outputs.baseline_all_times }} | |
| BASE_ALL_HEAPS: ${{ steps.compare.outputs.baseline_all_heaps }} | |
| PR_GROWTH_HEAPS: ${{ steps.compare.outputs.pr_growth_heaps }} | |
| PR_GROWTH_TIMES: ${{ steps.compare.outputs.pr_growth_times }} | |
| PR_GROWTH_SLOPE: ${{ steps.compare.outputs.pr_growth_slope }} | |
| PR_GROWTH_R2: ${{ steps.compare.outputs.pr_growth_r2 }} | |
| BASE_GROWTH_HEAPS: ${{ steps.compare.outputs.baseline_growth_heaps }} | |
| BASE_GROWTH_TIMES: ${{ steps.compare.outputs.baseline_growth_times }} | |
| BASE_GROWTH_SLOPE: ${{ steps.compare.outputs.baseline_growth_slope }} | |
| BASE_GROWTH_R2: ${{ steps.compare.outputs.baseline_growth_r2 }} | |
| GROWTH_SLOPE_DIFF: ${{ steps.compare.outputs.growth_slope_diff }} | |
| GROWTH_SLOPE_PCT: ${{ steps.compare.outputs.growth_slope_pct }} | |
| COMMIT_SHA: ${{ steps.pr-ref.outputs.sha || github.sha }} | |
| TABLE_PARALLELISM: ${{ steps.config.outputs.table_parallelism }} | |
| with: | |
| script: | | |
| const peak = process.env.PR_PEAK; | |
| const time = process.env.PR_TIME; | |
| const basePeak = process.env.BASELINE_PEAK; | |
| const baseTime = process.env.BASELINE_TIME; | |
| const baseSrc = process.env.BASELINE_SRC; | |
| const peakPct = process.env.PEAK_PCT; | |
| const timePct = process.env.TIME_PCT; | |
| const peakDiff = process.env.PEAK_DIFF; | |
| const timeDiff = process.env.TIME_DIFF; | |
| const runs = process.env.PR_RUNS || '1'; | |
| const prTimeSpread = process.env.PR_TIME_SPREAD; | |
| const prHeapSpread = process.env.PR_HEAP_SPREAD; | |
| const prAllTimes = process.env.PR_ALL_TIMES; | |
| const prAllHeaps = process.env.PR_ALL_HEAPS; | |
| const baseTimeSpread = process.env.BASE_TIME_SPREAD; | |
| const baseHeapSpread = process.env.BASE_HEAP_SPREAD; | |
| const baseAllTimes = process.env.BASE_ALL_TIMES; | |
| const baseAllHeaps = process.env.BASE_ALL_HEAPS; | |
| // Growth data | |
| const prGrowthHeaps = process.env.PR_GROWTH_HEAPS; | |
| const prGrowthTimes = process.env.PR_GROWTH_TIMES; | |
| const prGrowthSlope = process.env.PR_GROWTH_SLOPE; | |
| const prGrowthR2 = process.env.PR_GROWTH_R2; | |
| const baseGrowthHeaps = process.env.BASE_GROWTH_HEAPS; | |
| const baseGrowthTimes = process.env.BASE_GROWTH_TIMES; | |
| const baseGrowthSlope = process.env.BASE_GROWTH_SLOPE; | |
| const baseGrowthR2 = process.env.BASE_GROWTH_R2; | |
| const growthSlopeDiff = process.env.GROWTH_SLOPE_DIFF; | |
| const growthSlopePct = process.env.GROWTH_SLOPE_PCT; | |
| const fmt = (v) => parseFloat(v) >= 0 ? `+${v}` : v; | |
| const icon = (pct) => parseFloat(pct) > 5 ? '🔴' : parseFloat(pct) < -5 ? '🟢' : '⚪'; | |
| const SPREAD_THRESHOLD = 5.0; | |
| // --- Section 1: Primary benchmark --- | |
| const nLabel = parseInt(runs) > 1 ? ` (median of ${runs})` : ''; | |
| const tableParallelism = process.env.TABLE_PARALLELISM; | |
| const tpLabel = tableParallelism ? tableParallelism : 'auto (cores / 3)'; | |
| let body = `## Benchmark — ethrex 20 transfers${nLabel}\n\n`; | |
| body += `<sub>Table parallelism: ${tpLabel}</sub>\n\n`; | |
| body += `| Metric | main | PR | Δ |\n`; | |
| body += `|--------|------|----|---|\n`; | |
| body += `| **Peak heap** | ${basePeak} MB | ${peak} MB | ${fmt(peakDiff)} MB (${fmt(peakPct)}%) ${icon(peakPct)} |\n`; | |
| body += `| **Prove time** | ${baseTime}s | ${time}s | ${fmt(timeDiff)}s (${fmt(timePct)}%) ${icon(timePct)} |\n\n`; | |
| const regression = parseFloat(peakPct) > 5 || parseFloat(timePct) > 5; | |
| const improvement = parseFloat(peakPct) < -5 || parseFloat(timePct) < -5; | |
| if (regression) { | |
| body += `> ⚠️ **Regression detected** — heap or time increased by more than 5%.\n`; | |
| } else if (improvement) { | |
| body += `> 🎉 **Improvement detected** — heap or time decreased by more than 5%.\n`; | |
| } else { | |
| body += `> ✅ No significant change.\n`; | |
| } | |
| // Tier-1 -> Tier-2 escalation: a small time speedup the cheap 3-5 run CI | |
| // can't confirm (it catches >=~1.5% on its own). Point at the drift-free | |
| // ABBA tiebreaker, which the user runs on demand via `/bench-abba`. | |
| const tp = parseFloat(timePct); | |
| if (tp < 0 && tp > -1.5) { | |
| body += `>\n`; | |
| body += `> 🔬 **Looks like a small speedup (${fmt(timePct)}%) — below what ${runs} runs can confirm.** `; | |
| body += `Comment \`/bench-abba\` to run the drift-free ABBA tiebreaker (paired-t CI + exact Wilcoxon). `; | |
| body += `Note: it occupies the bench server for ~30–40 min.\n`; | |
| body += `> Optional pair count: \`/bench-abba 32\` (20 resolves ~1%, 32 for ~0.6%).\n`; | |
| } | |
| // Spread warnings | |
| const prWarnings = []; | |
| const baseWarnings = []; | |
| if (prTimeSpread && parseFloat(prTimeSpread) > SPREAD_THRESHOLD) { | |
| const vals = prAllTimes ? prAllTimes.split('/').map(t => `${t}s`).join(' / ') : ''; | |
| prWarnings.push(`Prove time spread: ${prTimeSpread}% (${vals})`); | |
| } | |
| if (prHeapSpread && parseFloat(prHeapSpread) > SPREAD_THRESHOLD) { | |
| const vals = prAllHeaps ? prAllHeaps.split('/').map(h => `${h} MB`).join(' / ') : ''; | |
| prWarnings.push(`Heap spread: ${prHeapSpread}% (${vals})`); | |
| } | |
| if (baseTimeSpread && parseFloat(baseTimeSpread) > SPREAD_THRESHOLD) { | |
| const vals = baseAllTimes ? baseAllTimes.split('/').map(t => `${t}s`).join(' / ') : ''; | |
| baseWarnings.push(`Baseline time spread: ${baseTimeSpread}% (${vals}) — comparison may be less reliable`); | |
| } | |
| if (baseHeapSpread && parseFloat(baseHeapSpread) > SPREAD_THRESHOLD) { | |
| const vals = baseAllHeaps ? baseAllHeaps.split('/').map(h => `${h} MB`).join(' / ') : ''; | |
| baseWarnings.push(`Baseline heap spread: ${baseHeapSpread}% (${vals}) — comparison may be less reliable`); | |
| } | |
| const allWarnings = [...prWarnings, ...baseWarnings]; | |
| if (allWarnings.length > 0) { | |
| body += `\n`; | |
| for (const w of allWarnings) { | |
| body += `> ⚠️ ${w}\n`; | |
| } | |
| if (prWarnings.length > 0) { | |
| body += `> Consider re-running \`/bench\`\n`; | |
| } | |
| } else if (parseInt(runs) > 1) { | |
| body += `\n> ✅ Low variance (time: ${prTimeSpread || '0.0'}%, heap: ${prHeapSpread || '0.0'}%)\n`; | |
| } | |
| // --- Section 2: Memory growth --- | |
| if (prGrowthHeaps) { | |
| const prHeaps = prGrowthHeaps.split('/'); | |
| const baseHeaps = baseGrowthHeaps ? baseGrowthHeaps.split('/') : null; | |
| // Transfer counts (x-axis); keep in sync with GROWTH_STEPS in the env block. | |
| const labels = ['4', '8', '12', '16', '20']; | |
| const n = prHeaps.length; | |
| body += `\n## Memory Growth\n\n`; | |
| body += `<sub>ethrex distinct-account transfers · default parallelism · 1 sample per point</sub>\n\n`; | |
| if (baseHeaps && baseHeaps.length === n && baseHeaps[0]) { | |
| body += `| Transfers | main (MB) | PR (MB) | Δ |\n`; | |
| body += `|-----------|-----------|---------|---|\n`; | |
| for (let i = 0; i < n; i++) { | |
| const bh = parseInt(baseHeaps[i]); | |
| const ph = parseInt(prHeaps[i]); | |
| const diff = ph - bh; | |
| const pct = bh > 0 ? ((diff / bh) * 100).toFixed(1) : '0.0'; | |
| body += `| ${labels[i]} | ${baseHeaps[i]} | ${prHeaps[i]} | ${fmt(diff)} MB (${fmt(pct)}%) |\n`; | |
| } | |
| } else { | |
| body += `| Transfers | PR (MB) |\n`; | |
| body += `|-----------|---------|\n`; | |
| for (let i = 0; i < n; i++) { | |
| body += `| ${labels[i]} | ${prHeaps[i]} |\n`; | |
| } | |
| } | |
| body += `\n`; | |
| if (prGrowthSlope) { | |
| body += `**Growth rate:** ${prGrowthSlope} MB / transfer`; | |
| if (baseGrowthSlope && growthSlopePct) { | |
| body += ` (main: ${baseGrowthSlope}, Δ: ${fmt(growthSlopePct)}%)`; | |
| } | |
| body += `\n`; | |
| } | |
| if (prGrowthR2) { | |
| body += `**Fit:** R² = ${prGrowthR2}`; | |
| if (baseGrowthR2) { | |
| body += ` (main: ${baseGrowthR2})`; | |
| } | |
| body += `\n`; | |
| } | |
| if (baseGrowthSlope && growthSlopePct) { | |
| const slopePctVal = parseFloat(growthSlopePct); | |
| if (slopePctVal > 10) { | |
| body += `\n> ⚠️ **Memory scaling regression** — growth rate increased by ${fmt(growthSlopePct)}%\n`; | |
| } else if (slopePctVal < -10) { | |
| body += `\n> 🎉 **Memory scaling improvement** — growth rate decreased by ${growthSlopePct}%\n`; | |
| } else { | |
| body += `\n> ✅ No significant change in memory scaling.\n`; | |
| } | |
| } | |
| } | |
| // --- Footer --- | |
| const sha = process.env.COMMIT_SHA.substring(0, 8); | |
| body += `\n<sub>Commit: ${sha} · Baseline: ${baseSrc} · Runner: self-hosted bench</sub>\n`; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| // Find existing comment (check both old and new markers for transition) | |
| const existing = comments.find(c => | |
| c.user.type === 'Bot' && ( | |
| c.body.includes('Benchmark — ethrex') || | |
| c.body.includes('Benchmark — fib_iterative_8M') || | |
| c.body.includes('Benchmark — fib_iterative_2M') || | |
| c.body.includes('Benchmark — fib_iterative_372k') | |
| ) | |
| ); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: existing.id, | |
| body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body | |
| }); | |
| } |