|
| 1 | +name: benchmark |
| 2 | + |
| 3 | +# Runs Criterion benchmarks on the terraphim-native runner (sccache-backed) |
| 4 | +# and enforces a regression gate: any benchmark that degrades >20% relative to |
| 5 | +# the stored baseline fails the job. |
| 6 | +# |
| 7 | +# Baseline is stored on the runner at ~/.cache/terraphim-bench/baseline.json |
| 8 | +# and updated whenever a push lands on main. |
| 9 | +# |
| 10 | +# The GitHub performance-benchmarking.yml is intentionally scoped to |
| 11 | +# workflow_dispatch only -- this workflow owns CI benchmarking. |
| 12 | + |
| 13 | +on: |
| 14 | + push: |
| 15 | + branches: [main] |
| 16 | + workflow_dispatch: |
| 17 | + |
| 18 | +env: |
| 19 | + CARGO_TERM_COLOR: always |
| 20 | + RUSTC_WRAPPER: /home/alex/.local/bin/sccache |
| 21 | + SCCACHE_BUCKET: rust-cache |
| 22 | + SCCACHE_SERVER_PORT: "4231" |
| 23 | + SCCACHE_ENDPOINT: http://172.26.0.1:8333 |
| 24 | + SCCACHE_S3_USE_SSL: "false" |
| 25 | + SCCACHE_REGION: us-east-1 |
| 26 | + SCCACHE_S3_KEY_PREFIX: terraphim-ai |
| 27 | + AWS_ACCESS_KEY_ID: any |
| 28 | + AWS_SECRET_ACCESS_KEY: any |
| 29 | + CARGO_INCREMENTAL: "0" |
| 30 | + |
| 31 | +jobs: |
| 32 | + criterion-benchmarks: |
| 33 | + name: Criterion Benchmarks + Regression Gate |
| 34 | + runs-on: terraphim-native |
| 35 | + |
| 36 | + steps: |
| 37 | + - name: sccache start and zero stats |
| 38 | + run: | |
| 39 | + /home/alex/.local/bin/sccache --start-server || true |
| 40 | + /home/alex/.local/bin/sccache --zero-stats |
| 41 | +
|
| 42 | + - name: Run Criterion benchmarks |
| 43 | + run: | |
| 44 | + mkdir -p benchmark-results |
| 45 | + cargo bench -p terraphim_tinyclaw --bench tinyclaw_benchmarks \ |
| 46 | + 2>&1 | tee benchmark-results/bench-output.txt |
| 47 | +
|
| 48 | + - name: Collect Criterion estimates |
| 49 | + run: | |
| 50 | + python3 - <<'PYEOF' |
| 51 | + import json, os, pathlib |
| 52 | + from datetime import datetime |
| 53 | +
|
| 54 | + today = datetime.now().strftime("%Y-%m-%d") |
| 55 | + p = pathlib.Path("target/criterion") |
| 56 | + results = {} |
| 57 | + if p.exists(): |
| 58 | + for f in sorted(p.glob("*/new/estimates.json")): |
| 59 | + name = f.parent.parent.name |
| 60 | + data = json.loads(f.read_text()) |
| 61 | + results[name] = data.get("mean", {}).get("point_estimate") |
| 62 | +
|
| 63 | + out = {"date": today, "estimates": results} |
| 64 | + out_path = f"benchmark-results/current-{today}.json" |
| 65 | + os.makedirs("benchmark-results", exist_ok=True) |
| 66 | + with open(out_path, "w") as fh: |
| 67 | + json.dump(out, fh, indent=2) |
| 68 | + count = len(results) |
| 69 | + names = list(results.keys()) |
| 70 | + print(f"Collected {count} benchmark(s): {names}") |
| 71 | + PYEOF |
| 72 | +
|
| 73 | + - name: Regression gate |
| 74 | + run: | |
| 75 | + BASELINE_STORE="${HOME}/.cache/terraphim-bench/baseline.json" |
| 76 | + CURRENT=$(ls benchmark-results/current-*.json 2>/dev/null | sort | tail -1) |
| 77 | +
|
| 78 | + if [ -z "${CURRENT}" ]; then |
| 79 | + echo "No Criterion output collected -- bench step may have failed" |
| 80 | + exit 1 |
| 81 | + fi |
| 82 | +
|
| 83 | + if [ ! -f "${BASELINE_STORE}" ]; then |
| 84 | + echo "No baseline found -- publishing current results as today's baseline" |
| 85 | + mkdir -p "$(dirname "${BASELINE_STORE}")" |
| 86 | + cp "${CURRENT}" "${BASELINE_STORE}" |
| 87 | + echo "Baseline written to ${BASELINE_STORE} ($(date +%Y-%m-%d))" |
| 88 | + exit 0 |
| 89 | + fi |
| 90 | +
|
| 91 | + python3 - "${BASELINE_STORE}" "${CURRENT}" <<'PYEOF' |
| 92 | + import json, sys |
| 93 | +
|
| 94 | + baseline = json.load(open(sys.argv[1]))["estimates"] |
| 95 | + current = json.load(open(sys.argv[2]))["estimates"] |
| 96 | +
|
| 97 | + regressions = [] |
| 98 | + for name, base_ns in baseline.items(): |
| 99 | + if base_ns is None or base_ns == 0: |
| 100 | + continue |
| 101 | + curr_ns = current.get(name) |
| 102 | + if curr_ns is None: |
| 103 | + print(f" MISSING {name} (baseline {base_ns:.1f} ns)") |
| 104 | + continue |
| 105 | + pct = (curr_ns - base_ns) / base_ns * 100 |
| 106 | + if pct > 20: |
| 107 | + regressions.append((name, pct, base_ns, curr_ns)) |
| 108 | + print(f" REGRESS {name}: +{pct:.1f}% {base_ns:.1f} -> {curr_ns:.1f} ns") |
| 109 | + else: |
| 110 | + print(f" ok {name}: {pct:+.1f}% {base_ns:.1f} -> {curr_ns:.1f} ns") |
| 111 | +
|
| 112 | + if regressions: |
| 113 | + print(f"\nFAIL: {len(regressions)} benchmark(s) regressed >20%") |
| 114 | + sys.exit(1) |
| 115 | + print("\nPASS: no regressions detected") |
| 116 | + PYEOF |
| 117 | +
|
| 118 | + - name: Update baseline on main |
| 119 | + if: github.ref == 'refs/heads/main' |
| 120 | + run: | |
| 121 | + CURRENT=$(ls benchmark-results/current-*.json 2>/dev/null | sort | tail -1) |
| 122 | + if [ -n "${CURRENT}" ]; then |
| 123 | + mkdir -p "${HOME}/.cache/terraphim-bench" |
| 124 | + cp "${CURRENT}" "${HOME}/.cache/terraphim-bench/baseline.json" |
| 125 | + echo "Baseline updated to $(date +%Y-%m-%d)" |
| 126 | + fi |
| 127 | +
|
| 128 | + - name: sccache stats |
| 129 | + if: always() |
| 130 | + run: /home/alex/.local/bin/sccache --show-stats |
0 commit comments