Skip to content

Commit 94513f0

Browse files
xangcastlexangcastle
committed
perf(benchmark): gate on Bazel analysis phase and report per-function Starlark cost
Rewrite the benchmark to gate on Bazel's analysis phase (runAnalysisPhase extracted from --profile) instead of process wall time, excluding JVM/IO noise. The two separate workspaces are merged into a single parameterized one where --packages drive both the analysis load and a fan-in py_binary whose venv scales with it, and the PR comment now surfaces a per-function Starlark CPU breakdown (via --starlark_cpu_profile) so that when something regresses it tells you which functions got slower not just that it did.
1 parent e12560c commit 94513f0

22 files changed

Lines changed: 948 additions & 748 deletions

‎.github/workflows/performance-benchmark.yml‎

Lines changed: 57 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ on:
77
workflow_dispatch:
88

99
env:
10-
BCR_VERSION: "1.11.7"
11-
ANALYSIS_BCR_VERSION: "2.0.0-alpha.4"
10+
BCR_VERSION: "2.0.0-alpha.4"
11+
PACKAGES: "50"
1212

1313
jobs:
1414
startup:
@@ -39,85 +39,67 @@ jobs:
3939
rm -f "$DEB"
4040
hyperfine --version
4141
42+
- name: Generate workspace
43+
run: |
44+
set -euo pipefail
45+
cd rules_py_pr/benchmark/workspace
46+
python3 generate_workspace.py --root . --packages "${PACKAGES}"
47+
4248
# ── BCR baseline ────────────────────────────────────────────────────────
4349
- name: Benchmark BCR ${{ env.BCR_VERSION }}
4450
run: |
4551
set -euo pipefail
46-
cd rules_py_pr/benchmark/startup
52+
cd rules_py_pr/benchmark/workspace
4753
python3 generate_module.py bcr --version "${BCR_VERSION}"
4854
4955
OUT_BASE="/tmp/bazel-bcr"
5056
rm -rf "$OUT_BASE"
51-
52-
START=$(date +%s%N)
53-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
54-
END=$(date +%s%N)
55-
BUILD_MS=$(( (END - START) / 1000000 ))
56-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/bcr-build.json"
57+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
5758
5859
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
59-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
60+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
6061
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/bcr.json" "$BIN"
61-
62-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
63-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
64-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/bcr-syspath.json"
62+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/bcr-syspath.json"
6563
6664
# ── HEAD main ───────────────────────────────────────────────────────────
6765
- name: Benchmark HEAD main
6866
run: |
6967
set -euo pipefail
70-
cd rules_py_pr/benchmark/startup
68+
cd rules_py_pr/benchmark/workspace
7169
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"
7270
7371
OUT_BASE="/tmp/bazel-main"
7472
rm -rf "$OUT_BASE"
75-
76-
START=$(date +%s%N)
77-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
78-
END=$(date +%s%N)
79-
BUILD_MS=$(( (END - START) / 1000000 ))
80-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/main-build.json"
73+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
8174
8275
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
83-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
76+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
8477
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/main.json" "$BIN"
85-
86-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
87-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
88-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/main-syspath.json"
78+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/main-syspath.json"
8979
9080
# ── Current commit (PR) ─────────────────────────────────────────────────
9181
- name: Benchmark current PR
9282
run: |
9383
set -euo pipefail
94-
cd rules_py_pr/benchmark/startup
84+
cd rules_py_pr/benchmark/workspace
9585
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"
9686
9787
OUT_BASE="/tmp/bazel-pr"
9888
rm -rf "$OUT_BASE"
99-
100-
START=$(date +%s%N)
101-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench //:bench_syspath
102-
END=$(date +%s%N)
103-
BUILD_MS=$(( (END - START) / 1000000 ))
104-
echo "{\"build_ms\": $BUILD_MS}" > "${GITHUB_WORKSPACE}/pr-build.json"
89+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= //:bench
10590
10691
BIN=$(bazel --output_base="$OUT_BASE" cquery //:bench --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
107-
test -x "$BIN" || { echo "ERROR: benchmark binary not executable: $BIN"; exit 1; }
92+
test -x "$BIN" || { echo "ERROR: bench binary not executable: $BIN"; exit 1; }
10893
RUNFILES_DIR="$PWD/$BIN.runfiles" hyperfine --warmup 5 --runs 50 --export-json "${GITHUB_WORKSPACE}/pr.json" "$BIN"
109-
110-
BIN_SP=$(bazel --output_base="$OUT_BASE" cquery //:bench_syspath --disk_cache= --output=starlark --starlark:expr='target.files_to_run.executable.path' | tail -n1)
111-
test -x "$BIN_SP" || { echo "ERROR: bench_syspath binary not executable: $BIN_SP"; exit 1; }
112-
RUNFILES_DIR="$PWD/$BIN_SP.runfiles" "$BIN_SP" "$GITHUB_WORKSPACE/pr-syspath.json"
94+
RUNFILES_DIR="$PWD/$BIN.runfiles" "$BIN" "${GITHUB_WORKSPACE}/pr-syspath.json"
11395
11496
# ── Compare ─────────────────────────────────────────────────────────────
11597
- name: Compare startup results
11698
id: compare
11799
continue-on-error: true
118100
run: |
119101
set -euo pipefail
120-
python3 rules_py_pr/benchmark/startup/compare.py \
102+
python3 rules_py_pr/benchmark/compare.py startup \
121103
--output-table startup-table.txt \
122104
bcr.json main.json pr.json
123105
@@ -156,84 +138,79 @@ jobs:
156138
ref: main
157139
path: rules_py_main
158140

159-
- name: Install hyperfine
160-
run: |
161-
set -euo pipefail
162-
HYPERFINE_VERSION="1.18.0"
163-
DEB="hyperfine_${HYPERFINE_VERSION}_amd64.deb"
164-
URL="https://github.com/sharkdp/hyperfine/releases/download/v${HYPERFINE_VERSION}/${DEB}"
165-
wget -qO "$DEB" "$URL"
166-
sudo dpkg -i "$DEB"
167-
rm -f "$DEB"
168-
hyperfine --version
169-
170141
- name: Generate workspace
171142
run: |
172143
set -euo pipefail
173-
cd rules_py_pr/benchmark/analysis
174-
python3 workspace/generate_workspace.py --root workspace --packages 50
144+
cd rules_py_pr/benchmark/workspace
145+
python3 generate_workspace.py --root . --packages "${PACKAGES}"
175146
176147
# ── BCR baseline ────────────────────────────────────────────────────────
177-
- name: Benchmark BCR ${{ env.ANALYSIS_BCR_VERSION }}
148+
- name: Benchmark BCR ${{ env.BCR_VERSION }}
178149
run: |
179150
set -euo pipefail
180-
cd rules_py_pr/benchmark/analysis
181-
python3 generate_module.py bcr --version "${ANALYSIS_BCR_VERSION}"
151+
cd rules_py_pr/benchmark/workspace
152+
python3 generate_module.py bcr --version "${BCR_VERSION}"
182153
183154
OUT_BASE="/tmp/bazel-bcr"
184155
rm -rf "$OUT_BASE"
156+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
185157
186-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
187-
188-
hyperfine --warmup 1 --runs 10 \
158+
python3 ../profile_benchmark.py \
159+
--runs 10 --warmup 1 \
189160
--prepare 'rm -rf /tmp/bazel-bcr-analysis' \
190-
--export-json "${GITHUB_WORKSPACE}/bcr.json" \
191-
'bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
161+
--output "${GITHUB_WORKSPACE}/bcr.json" \
162+
--save-profile "${GITHUB_WORKSPACE}/profiles/bcr-profile.gz" \
163+
--save-starlark "${GITHUB_WORKSPACE}/profiles/bcr-starlark.pprof.gz" \
164+
-- bazel --output_base=/tmp/bazel-bcr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
192165
193-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/bcr-packages.txt
194-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/bcr-targets.txt
166+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/bcr-packages.txt
167+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/bcr-targets.txt
195168
echo "{\"packages\": $(wc -l < /tmp/bcr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/bcr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/bcr-aux.json"
196169
197170
# ── HEAD main ───────────────────────────────────────────────────────────
198171
- name: Benchmark HEAD main
199172
run: |
200173
set -euo pipefail
201-
cd rules_py_pr/benchmark/analysis
174+
cd rules_py_pr/benchmark/workspace
202175
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_main"
203176
204177
OUT_BASE="/tmp/bazel-main"
205178
rm -rf "$OUT_BASE"
179+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
206180
207-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
208-
209-
hyperfine --warmup 1 --runs 10 \
181+
python3 ../profile_benchmark.py \
182+
--runs 10 --warmup 1 \
210183
--prepare 'rm -rf /tmp/bazel-main-analysis' \
211-
--export-json "${GITHUB_WORKSPACE}/main.json" \
212-
'bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
184+
--output "${GITHUB_WORKSPACE}/main.json" \
185+
--save-profile "${GITHUB_WORKSPACE}/profiles/main-profile.gz" \
186+
--save-starlark "${GITHUB_WORKSPACE}/profiles/main-starlark.pprof.gz" \
187+
-- bazel --output_base=/tmp/bazel-main-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
213188
214-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/main-packages.txt
215-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/main-targets.txt
189+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/main-packages.txt
190+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/main-targets.txt
216191
echo "{\"packages\": $(wc -l < /tmp/main-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/main-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/main-aux.json"
217192
218193
# ── Current commit (PR) ─────────────────────────────────────────────────
219194
- name: Benchmark current PR
220195
run: |
221196
set -euo pipefail
222-
cd rules_py_pr/benchmark/analysis
197+
cd rules_py_pr/benchmark/workspace
223198
python3 generate_module.py local --path "$GITHUB_WORKSPACE/rules_py_pr"
224199
225200
OUT_BASE="/tmp/bazel-pr"
226201
rm -rf "$OUT_BASE"
202+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //...
227203
228-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc fetch //workspace/...
229-
230-
hyperfine --warmup 1 --runs 10 \
204+
python3 ../profile_benchmark.py \
205+
--runs 10 --warmup 1 \
231206
--prepare 'rm -rf /tmp/bazel-pr-analysis' \
232-
--export-json "${GITHUB_WORKSPACE}/pr.json" \
233-
'bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild //workspace/...'
207+
--output "${GITHUB_WORKSPACE}/pr.json" \
208+
--save-profile "${GITHUB_WORKSPACE}/profiles/pr-profile.gz" \
209+
--save-starlark "${GITHUB_WORKSPACE}/profiles/pr-starlark.pprof.gz" \
210+
-- bazel --output_base=/tmp/bazel-pr-analysis --bazelrc=../../.github/workflows/ci.bazelrc build --disk_cache= --nobuild --profile={PROFILE} --starlark_cpu_profile={STARPROFILE} //...
234211
235-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... --output=package > /tmp/pr-packages.txt
236-
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //workspace/... > /tmp/pr-targets.txt
212+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... --output=package > /tmp/pr-packages.txt
213+
bazel --output_base="$OUT_BASE" --bazelrc=../../.github/workflows/ci.bazelrc query //... > /tmp/pr-targets.txt
237214
echo "{\"packages\": $(wc -l < /tmp/pr-packages.txt | tr -d ' '), \"targets\": $(wc -l < /tmp/pr-targets.txt | tr -d ' ')}" > "${GITHUB_WORKSPACE}/pr-aux.json"
238215
239216
# ── Compare ─────────────────────────────────────────────────────────────
@@ -242,7 +219,7 @@ jobs:
242219
continue-on-error: true
243220
run: |
244221
set -euo pipefail
245-
python3 rules_py_pr/benchmark/analysis/compare.py \
222+
python3 rules_py_pr/benchmark/compare.py analysis \
246223
--output-table analysis-table.txt \
247224
bcr.json main.json pr.json
248225
@@ -253,6 +230,7 @@ jobs:
253230
name: analysis-benchmark-results
254231
path: |
255232
analysis-table.txt
233+
profiles/
256234
if-no-files-found: warn
257235

258236
- name: Fail on regression

‎benchmark/analysis/.bazelrc‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎benchmark/analysis/MODULE.bazel.template‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

0 commit comments

Comments
 (0)