Skip to content

Commit 79df0c6

Browse files
author
Test User
committed
ci(bench): port performance-benchmarking to Gitea native-ci, scope GitHub to manual
Add .gitea/workflows/benchmark.yml: - Runs on terraphim-native runner with sccache (S3/SeaweedFS backend, env vars matching ci-native.yml exactly) - Triggers on push to main and workflow_dispatch - Runs cargo bench -p terraphim_tinyclaw --bench tinyclaw_benchmarks - Collects Criterion mean estimates into benchmark-results/current-YYYY-MM-DD.json - Regression gate compares against runner-local baseline at ~/.cache/terraphim-bench/baseline.json; fails if any benchmark degrades >20%; seeds baseline from first run (today's date) - Updates baseline automatically on main-branch pushes Scope .github/workflows/performance-benchmarking.yml to manual only: - Remove pull_request: and push: triggers; retain workflow_dispatch: - Add comment explaining the split: Gitea handles CI benchmarks, GitHub workflow kept for deep on-demand analysis and SLO reporting
1 parent 98fa93b commit 79df0c6

2 files changed

Lines changed: 135 additions & 12 deletions

File tree

.gitea/workflows/benchmark.yml

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: benchmark
2+
3+
# Runs Criterion benchmarks on the terraphim-native runner (sccache-backed)
4+
# and enforces a regression gate: any benchmark that degrades >20% relative to
5+
# the stored baseline fails the job.
6+
#
7+
# Baseline is stored on the runner at ~/.cache/terraphim-bench/baseline.json
8+
# and updated whenever a push lands on main.
9+
#
10+
# The GitHub performance-benchmarking.yml is intentionally scoped to
11+
# workflow_dispatch only -- this workflow owns CI benchmarking.
12+
13+
on:
14+
push:
15+
branches: [main]
16+
workflow_dispatch:
17+
18+
env:
19+
CARGO_TERM_COLOR: always
20+
RUSTC_WRAPPER: /home/alex/.local/bin/sccache
21+
SCCACHE_BUCKET: rust-cache
22+
SCCACHE_SERVER_PORT: "4231"
23+
SCCACHE_ENDPOINT: http://172.26.0.1:8333
24+
SCCACHE_S3_USE_SSL: "false"
25+
SCCACHE_REGION: us-east-1
26+
SCCACHE_S3_KEY_PREFIX: terraphim-ai
27+
AWS_ACCESS_KEY_ID: any
28+
AWS_SECRET_ACCESS_KEY: any
29+
CARGO_INCREMENTAL: "0"
30+
31+
jobs:
32+
criterion-benchmarks:
33+
name: Criterion Benchmarks + Regression Gate
34+
runs-on: terraphim-native
35+
36+
steps:
37+
- name: sccache start and zero stats
38+
run: |
39+
/home/alex/.local/bin/sccache --start-server || true
40+
/home/alex/.local/bin/sccache --zero-stats
41+
42+
- name: Run Criterion benchmarks
43+
run: |
44+
mkdir -p benchmark-results
45+
cargo bench -p terraphim_tinyclaw --bench tinyclaw_benchmarks \
46+
2>&1 | tee benchmark-results/bench-output.txt
47+
48+
- name: Collect Criterion estimates
49+
run: |
50+
python3 - <<'PYEOF'
51+
import json, os, pathlib
52+
from datetime import datetime
53+
54+
today = datetime.now().strftime("%Y-%m-%d")
55+
p = pathlib.Path("target/criterion")
56+
results = {}
57+
if p.exists():
58+
for f in sorted(p.glob("*/new/estimates.json")):
59+
name = f.parent.parent.name
60+
data = json.loads(f.read_text())
61+
results[name] = data.get("mean", {}).get("point_estimate")
62+
63+
out = {"date": today, "estimates": results}
64+
out_path = f"benchmark-results/current-{today}.json"
65+
os.makedirs("benchmark-results", exist_ok=True)
66+
with open(out_path, "w") as fh:
67+
json.dump(out, fh, indent=2)
68+
count = len(results)
69+
names = list(results.keys())
70+
print(f"Collected {count} benchmark(s): {names}")
71+
PYEOF
72+
73+
- name: Regression gate
74+
run: |
75+
BASELINE_STORE="${HOME}/.cache/terraphim-bench/baseline.json"
76+
CURRENT=$(ls benchmark-results/current-*.json 2>/dev/null | sort | tail -1)
77+
78+
if [ -z "${CURRENT}" ]; then
79+
echo "No Criterion output collected -- bench step may have failed"
80+
exit 1
81+
fi
82+
83+
if [ ! -f "${BASELINE_STORE}" ]; then
84+
echo "No baseline found -- publishing current results as today's baseline"
85+
mkdir -p "$(dirname "${BASELINE_STORE}")"
86+
cp "${CURRENT}" "${BASELINE_STORE}"
87+
echo "Baseline written to ${BASELINE_STORE} ($(date +%Y-%m-%d))"
88+
exit 0
89+
fi
90+
91+
python3 - "${BASELINE_STORE}" "${CURRENT}" <<'PYEOF'
92+
import json, sys
93+
94+
baseline = json.load(open(sys.argv[1]))["estimates"]
95+
current = json.load(open(sys.argv[2]))["estimates"]
96+
97+
regressions = []
98+
for name, base_ns in baseline.items():
99+
if base_ns is None or base_ns == 0:
100+
continue
101+
curr_ns = current.get(name)
102+
if curr_ns is None:
103+
print(f" MISSING {name} (baseline {base_ns:.1f} ns)")
104+
continue
105+
pct = (curr_ns - base_ns) / base_ns * 100
106+
if pct > 20:
107+
regressions.append((name, pct, base_ns, curr_ns))
108+
print(f" REGRESS {name}: +{pct:.1f}% {base_ns:.1f} -> {curr_ns:.1f} ns")
109+
else:
110+
print(f" ok {name}: {pct:+.1f}% {base_ns:.1f} -> {curr_ns:.1f} ns")
111+
112+
if regressions:
113+
print(f"\nFAIL: {len(regressions)} benchmark(s) regressed >20%")
114+
sys.exit(1)
115+
print("\nPASS: no regressions detected")
116+
PYEOF
117+
118+
- name: Update baseline on main
119+
if: github.ref == 'refs/heads/main'
120+
run: |
121+
CURRENT=$(ls benchmark-results/current-*.json 2>/dev/null | sort | tail -1)
122+
if [ -n "${CURRENT}" ]; then
123+
mkdir -p "${HOME}/.cache/terraphim-bench"
124+
cp "${CURRENT}" "${HOME}/.cache/terraphim-bench/baseline.json"
125+
echo "Baseline updated to $(date +%Y-%m-%d)"
126+
fi
127+
128+
- name: sccache stats
129+
if: always()
130+
run: /home/alex/.local/bin/sccache --show-stats

.github/workflows/performance-benchmarking.yml

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
name: Performance Benchmarking
2+
# Scoped to manual (workflow_dispatch) only.
3+
# CI benchmarking and regression gate are handled by .gitea/workflows/benchmark.yml
4+
# on the terraphim-native runner. This workflow is retained for deep on-demand
5+
# analysis: arbitrary iteration counts, explicit baseline-ref comparison,
6+
# full SLO report generation, and artifact publishing to GitHub Actions.
27

38
on:
49
workflow_dispatch:
@@ -13,18 +18,6 @@ on:
1318
required: false
1419
default: 'main'
1520
type: string
16-
pull_request:
17-
paths:
18-
- 'crates/terraphim_*/src/**'
19-
- 'terraphim_server/src/**'
20-
- 'scripts/run-performance-benchmarks.sh'
21-
- '.github/workflows/performance-benchmarking.yml'
22-
push:
23-
branches: [main, develop]
24-
paths:
25-
- 'crates/terraphim_*/src/**'
26-
- 'terraphim_server/src/**'
27-
- 'scripts/run-performance-benchmarks.sh'
2821

2922
env:
3023
CARGO_TERM_COLOR: always

0 commit comments

Comments
 (0)