Skip to content

Commit 6db41ca

Browse files
xangcastlexangcastle
committed
perf(benchmark): gate on Bazel analysis phase and report per-function Starlark cost
Rewrite the benchmark to gate on Bazel's analysis phase (runAnalysisPhase extracted from --profile) instead of process wall time, excluding JVM/IO noise. The two separate workspaces are merged into a single parameterized one where --packages drive both the analysis load and a fan-in py_binary whose venv scales with it, and the PR comment now surfaces a per-function Starlark CPU breakdown (via --starlark_cpu_profile) so that when something regresses it tells you which functions got slower not just that it did.
1 parent e12560c commit 6db41ca

22 files changed

Lines changed: 916 additions & 748 deletions

‎.github/workflows/performance-benchmark.yml‎

Lines changed: 50 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ on:
77
workflow_dispatch:
88

99
env:
10-
BCR_VERSION: "1.11.7"
11-
ANALYSIS_BCR_VERSION: "2.0.0-alpha.4"
10+
BCR_VERSION: "2.0.0-alpha.4"
11+
PACKAGES: "50"
1212

1313
jobs:
1414
startup:
@@ -39,85 +39,67 @@ jobs:
3939
rm -f "$DEB"
4040
hyperfine --version
4141
42+
- name: Generate workspace
43+
run: |
44+
set -euo pipefail
45+
cd rules_py_pr/benchmark/workspace
46+
python3 generate_workspace.py --root . --packages "${PACKAGES}"
47+
4248
# ── BCR baseline ────────────────────────────────────────────────────────
4349
- name: Benchmark BCR ${{ env.BCR_VERSION }}
4450
run: |
4551
set -euo pipefail
46-
cd rules_py_pr/benchmark/startup
52+
cd rules_py_pr/benchmark/workspace
4753
python3 generate_module.py bcr --version "${BCR_VERSION}"
4854
4955
OUT_BASE="/tmp/bazel-bcr"
5056
rm -rf "$OUT_BASE"
51-
52-
START=$(date +%s%N)
53-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
54-
END=$(date +%s%N)
55-
BUILD_MS=$(( (END - START) / 1000000 ))
56-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/bcr-build.json"
57+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
5758
5859
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
59-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
60+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
6061
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/bcr.json" "$BIN"
61-
62-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
63-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
64-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/bcr-syspath.json"
62+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/bcr-syspath.json"
6563
6664
# ── HEAD main ───────────────────────────────────────────────────────────
6765
- name: Benchmark HEAD main
6866
run: |
6967
set -euo pipefail
70-
cd rules_py_pr/benchmark/startup
68+
cd rules_py_pr/benchmark/workspace
7169
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"
7270
7371
OUT_BASE="/tmp/bazel-main"
7472
rm -rf "$OUT_BASE"
75-
76-
START=$(date +%s%N)
77-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
78-
END=$(date +%s%N)
79-
BUILD_MS=$(( (END - START) / 1000000 ))
80-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/main-build.json"
73+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
8174
8275
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
83-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
76+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
8477
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/main.json" "$BIN"
85-
86-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
87-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
88-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/main-syspath.json"
78+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/main-syspath.json"
8979
9080
# ── Current commit (PR) ─────────────────────────────────────────────────
9181
- name: Benchmark current PR
9282
run: |
9383
set -euo pipefail
94-
cd rules_py_pr/benchmark/startup
84+
cd rules_py_pr/benchmark/workspace
9585
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"
9686
9787
OUT_BASE="/tmp/bazel-pr"
9888
rm -rf "$OUT_BASE"
99-
100-
START=$(date +%s%N)
101-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
102-
END=$(date +%s%N)
103-
BUILD_MS=$(( (END - START) / 1000000 ))
104-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/pr-build.json"
89+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
10590
10691
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
107-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
92+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
10893
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/pr.json" "$BIN"
109-
110-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
111-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
112-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/pr-syspath.json"
94+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/pr-syspath.json"
11395
11496
# ── Compare ─────────────────────────────────────────────────────────────
11597
- name: Compare startup results
11698
id: compare
11799
continue-on-error: true
118100
run: |
119101
set -euo pipefail
120-
python3 rules_py_pr/benchmark/startup/compare.py \
102+
python3 rules_py_pr/benchmark/compare.py startup \
121103
--output-table startup-table.txt \
122104
bcr.json main.json pr.json
123105
@@ -156,84 +138,73 @@ jobs:
156138
ref: main
157139
path: rules_py_main
158140

159-
- name: Install hyperfine
160-
run: |
161-
set -euo pipefail
162-
HYPERFINE_VERSION="1.18.0"
163-
DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb"
164-
URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}"
165-
wget -qO "$DEB" "$URL"
166-
sudo dpkg -i "$DEB"
167-
rm -f "$DEB"
168-
hyperfine --version
169-
170141
- name: Generate workspace
171142
run: |
172143
set -euo pipefail
173-
cd rules_py_pr/benchmark/analysis
174-
python3 workspace/generate_workspace.py --root workspace --packages 50
144+
cd rules_py_pr/benchmark/workspace
145+
python3 generate_workspace.py --root . --packages "${PACKAGES}"
175146
176147
# ── BCR baseline ────────────────────────────────────────────────────────
177-
- name: Benchmark BCR ${{ env.ANALYSIS_BCR_VERSION }}
148+
- name: Benchmark BCR ${{ env.BCR_VERSION }}
178149
run: |
179150
set -euo pipefail
180-
cd rules_py_pr/benchmark/analysis
181-
python3 generate_module.py bcr --version "${ANALYSIS_BCR_VERSION}"
151+
cd rules_py_pr/benchmark/workspace
152+
python3 generate_module.py bcr --version "${BCR_VERSION}"
182153
183154
OUT_BASE="/tmp/bazel-bcr"
184155
rm -rf "$OUT_BASE"
156+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
185157
186-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
187-
188-
hyperfine --warmup 1 --runs 10 \
158+
python3 ../profile_benchmark.py \
159+
--runs 10 --warmup 1 \
189160
--prepare 'rm -rf /tmp/bazel-bcr-analysis' \
190-
--export-json "${GITHUB_WORKSPACE}/bcr.json" \
191-
'bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
161+
--output "${GITHUB_WORKSPACE}/bcr.json" \
162+
-- bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
192163
193-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/bcr-packages.txt
194-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/bcr-targets.txt
164+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/bcr-packages.txt
165+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/bcr-targets.txt
195166
echo "{\"packages\": $(wc -l < /tmp/bcr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/bcr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/bcr-aux.json"
196167
197168
# ── HEAD main ───────────────────────────────────────────────────────────
198169
- name: Benchmark HEAD main
199170
run: |
200171
set -euo pipefail
201-
cd rules_py_pr/benchmark/analysis
172+
cd rules_py_pr/benchmark/workspace
202173
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"
203174
204175
OUT_BASE="/tmp/bazel-main"
205176
rm -rf "$OUT_BASE"
177+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
206178
207-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
208-
209-
hyperfine --warmup 1 --runs 10 \
179+
python3 ../profile_benchmark.py \
180+
--runs 10 --warmup 1 \
210181
--prepare 'rm -rf /tmp/bazel-main-analysis' \
211-
--export-json "${GITHUB_WORKSPACE}/main.json" \
212-
'bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
182+
--output "${GITHUB_WORKSPACE}/main.json" \
183+
-- bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
213184
214-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/main-packages.txt
215-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/main-targets.txt
185+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/main-packages.txt
186+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/main-targets.txt
216187
echo "{\"packages\": $(wc -l < /tmp/main-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/main-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/main-aux.json"
217188
218189
# ── Current commit (PR) ─────────────────────────────────────────────────
219190
- name: Benchmark current PR
220191
run: |
221192
set -euo pipefail
222-
cd rules_py_pr/benchmark/analysis
193+
cd rules_py_pr/benchmark/workspace
223194
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"
224195
225196
OUT_BASE="/tmp/bazel-pr"
226197
rm -rf "$OUT_BASE"
198+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
227199
228-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
229-
230-
hyperfine --warmup 1 --runs 10 \
200+
python3 ../profile_benchmark.py \
201+
--runs 10 --warmup 1 \
231202
--prepare 'rm -rf /tmp/bazel-pr-analysis' \
232-
--export-json "${GITHUB_WORKSPACE}/pr.json" \
233-
'bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
203+
--output "${GITHUB_WORKSPACE}/pr.json" \
204+
-- bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
234205
235-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/pr-packages.txt
236-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/pr-targets.txt
206+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/pr-packages.txt
207+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/pr-targets.txt
237208
echo "{\"packages\": $(wc -l < /tmp/pr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/pr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/pr-aux.json"
238209
239210
# ── Compare ─────────────────────────────────────────────────────────────
@@ -242,7 +213,7 @@ jobs:
242213
continue-on-error: true
243214
run: |
244215
set -euo pipefail
245-
python3 rules_py_pr/benchmark/analysis/compare.py \
216+
python3 rules_py_pr/benchmark/compare.py analysis \
246217
--output-table analysis-table.txt \
247218
bcr.json main.json pr.json
248219

‎benchmark/analysis/.bazelrc‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎benchmark/analysis/MODULE.bazel.template‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

0 commit comments

Comments
 (0)