Skip to content

Commit f4e5225

Browse files
BartWaardenburgBartWaardenburg
committed
test: broaden codspeed benchmark coverage
1 parent 2ea9902 commit f4e5225

14 files changed

Lines changed: 1355 additions & 125 deletions

File tree

‎.github/scripts/generate-benchmark-matrix.mjs‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { appendFileSync, readFileSync } from "node:fs";
2+
import { execFileSync } from "node:child_process";
3+
4+
const rustShard = ({
5+
label,
6+
cacheKey,
7+
packageName,
8+
bench,
9+
features = "codspeed",
10+
fixtures = false,
11+
}) => ({
12+
kind: "rust",
13+
mode: "simulation",
14+
label,
15+
cache_key: cacheKey,
16+
package: packageName,
17+
bench,
18+
features,
19+
fixtures,
20+
});
21+
22+
const jsShard = ({ label, cacheKey, command, fixtures = true }) => ({
23+
kind: "node",
24+
mode: "walltime",
25+
label,
26+
cache_key: cacheKey,
27+
command,
28+
fixtures,
29+
});
30+
31+
const SHARDS = {
32+
codec: rustShard({
33+
label: "codec vlq",
34+
cacheKey: "codec-vlq",
35+
packageName: "srcmap-codec",
36+
bench: "vlq",
37+
}),
38+
codecParallel: rustShard({
39+
label: "codec vlq parallel",
40+
cacheKey: "codec-vlq-parallel",
41+
packageName: "srcmap-codec",
42+
bench: "vlq",
43+
features: "codspeed,parallel",
44+
}),
45+
sourcemap: rustShard({
46+
label: "sourcemap parse",
47+
cacheKey: "sourcemap-parse",
48+
packageName: "srcmap-sourcemap",
49+
bench: "parse",
50+
fixtures: true,
51+
}),
52+
generator: rustShard({
53+
label: "generator",
54+
cacheKey: "generator",
55+
packageName: "srcmap-generator",
56+
bench: "generate",
57+
}),
58+
generatorParallel: rustShard({
59+
label: "generator parallel",
60+
cacheKey: "generator-parallel",
61+
packageName: "srcmap-generator",
62+
bench: "generate",
63+
features: "codspeed,parallel",
64+
}),
65+
remapping: rustShard({
66+
label: "remapping",
67+
cacheKey: "remapping",
68+
packageName: "srcmap-remapping",
69+
bench: "remap",
70+
fixtures: true,
71+
}),
72+
packages: jsShard({
73+
label: "package runtime",
74+
cacheKey: "package-runtime",
75+
command: "corepack pnpm --dir benchmarks run bench:codspeed:packages",
76+
}),
77+
};
78+
79+
const allShards = () => [
80+
SHARDS.codec,
81+
SHARDS.codecParallel,
82+
SHARDS.sourcemap,
83+
SHARDS.generator,
84+
SHARDS.generatorParallel,
85+
SHARDS.remapping,
86+
SHARDS.packages,
87+
];
88+
89+
const commandOutput = (command, args) => {
90+
try {
91+
return execFileSync(command, args, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
92+
} catch {
93+
return "";
94+
}
95+
};
96+
97+
const changedFilesForEvent = () => {
98+
const eventName = process.env.GITHUB_EVENT_NAME ?? "";
99+
if (eventName === "workflow_dispatch") return null;
100+
101+
if (eventName === "pull_request") {
102+
const baseRef = process.env.GITHUB_BASE_REF;
103+
if (!baseRef) return null;
104+
commandOutput("git", ["fetch", "--no-tags", "--depth=1", "origin", baseRef]);
105+
const diff = commandOutput("git", ["diff", "--name-only", `origin/${baseRef}...HEAD`]);
106+
return diff.trim() ? diff.trim().split("\n") : [];
107+
}
108+
109+
if (eventName === "push") {
110+
const eventPath = process.env.GITHUB_EVENT_PATH;
111+
if (!eventPath) return null;
112+
const event = JSON.parse(readFileSync(eventPath, "utf8"));
113+
const before = event.before;
114+
const after = event.after ?? process.env.GITHUB_SHA;
115+
if (!before || /^0+$/.test(before) || !after) return null;
116+
const diff = commandOutput("git", ["diff", "--name-only", `${before}..${after}`]);
117+
return diff.trim() ? diff.trim().split("\n") : [];
118+
}
119+
120+
return null;
121+
};
122+
123+
const includeShard = (selected, shard) => {
124+
selected.set(shard.label, shard);
125+
};
126+
127+
const selectShards = (files) => {
128+
if (files === null) return allShards();
129+
130+
const selected = new Map();
131+
132+
for (const file of files) {
133+
if (
134+
file === ".github/workflows/bench.yml" ||
135+
file === ".github/scripts/generate-benchmark-matrix.mjs" ||
136+
file === "Cargo.toml" ||
137+
file === "Cargo.lock" ||
138+
file === "package.json" ||
139+
file === "pnpm-lock.yaml" ||
140+
file === "pnpm-workspace.yaml"
141+
) {
142+
return allShards();
143+
}
144+
145+
if (file.startsWith("crates/codec/")) {
146+
includeShard(selected, SHARDS.codec);
147+
includeShard(selected, SHARDS.codecParallel);
148+
}
149+
if (file.startsWith("crates/sourcemap/")) includeShard(selected, SHARDS.sourcemap);
150+
if (file.startsWith("crates/generator/")) {
151+
includeShard(selected, SHARDS.generator);
152+
includeShard(selected, SHARDS.generatorParallel);
153+
}
154+
if (file.startsWith("crates/remapping/")) includeShard(selected, SHARDS.remapping);
155+
if (file.startsWith("benchmarks/download-fixtures.mjs")) {
156+
includeShard(selected, SHARDS.sourcemap);
157+
includeShard(selected, SHARDS.remapping);
158+
includeShard(selected, SHARDS.packages);
159+
}
160+
if (
161+
file.startsWith("benchmarks/") ||
162+
file.startsWith("packages/") ||
163+
file === "benchmarks/package.json"
164+
) {
165+
includeShard(selected, SHARDS.packages);
166+
}
167+
}
168+
169+
return selected.size === 0 ? allShards() : [...selected.values()];
170+
};
171+
172+
const include = selectShards(changedFilesForEvent());
173+
const json = JSON.stringify(include);
174+
175+
if (process.env.GITHUB_OUTPUT) {
176+
appendFileSync(process.env.GITHUB_OUTPUT, `include=${json}\n`);
177+
} else {
178+
console.log(json);
179+
}

‎.github/workflows/bench.yml‎

Lines changed: 91 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,81 +5,144 @@ on:
55
branches: [main]
66
paths:
77
- 'crates/**'
8-
- 'benchmarks/download-fixtures.mjs'
8+
- 'benchmarks/**'
9+
- 'packages/**'
910
- 'Cargo.toml'
1011
- 'Cargo.lock'
12+
- 'package.json'
13+
- 'pnpm-lock.yaml'
14+
- 'pnpm-workspace.yaml'
15+
- '.github/scripts/generate-benchmark-matrix.mjs'
1116
- '.github/workflows/bench.yml'
1217
push:
1318
branches: [main]
1419
paths:
1520
- 'crates/**'
16-
- 'benchmarks/download-fixtures.mjs'
21+
- 'benchmarks/**'
22+
- 'packages/**'
1723
- 'Cargo.toml'
1824
- 'Cargo.lock'
25+
- 'package.json'
26+
- 'pnpm-lock.yaml'
27+
- 'pnpm-workspace.yaml'
28+
- '.github/scripts/generate-benchmark-matrix.mjs'
1929
- '.github/workflows/bench.yml'
2030
workflow_dispatch:
2131

2232
concurrency:
2333
group: bench-${{ github.ref }}
2434
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
2535

26-
permissions: {}
36+
permissions:
37+
contents: read
38+
id-token: write
2739

2840
env:
2941
CARGO_TERM_COLOR: always
42+
RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed"
3043

3144
jobs:
45+
benchmark-matrix:
46+
name: Generate benchmark matrix
47+
runs-on: ubuntu-latest
48+
outputs:
49+
include: ${{ steps.matrix.outputs.include }}
50+
steps:
51+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
52+
with:
53+
fetch-depth: 0
54+
55+
- name: Generate benchmark matrix
56+
id: matrix
57+
run: node .github/scripts/generate-benchmark-matrix.mjs
58+
3259
benchmark:
33-
name: CodSpeed simulation (${{ matrix.label }})
60+
name: CodSpeed (${{ matrix.label }})
61+
needs: benchmark-matrix
3462
runs-on: ubuntu-latest
35-
timeout-minutes: 30
63+
timeout-minutes: 45
3664
strategy:
3765
fail-fast: false
3866
matrix:
39-
include:
40-
- label: codec vlq
41-
cache_key: codec-vlq
42-
package: srcmap-codec
43-
bench: vlq
44-
fixtures: false
45-
- label: sourcemap parse
46-
cache_key: sourcemap-parse
47-
package: srcmap-sourcemap
48-
bench: parse
49-
fixtures: true
50-
- label: generator
51-
cache_key: generator
52-
package: srcmap-generator
53-
bench: generate
54-
fixtures: false
55-
- label: remapping
56-
cache_key: remapping
57-
package: srcmap-remapping
58-
bench: remap
59-
fixtures: false
67+
include: ${{ fromJson(needs.benchmark-matrix.outputs.include) }}
6068
steps:
6169
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
6270

71+
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
72+
if: matrix.kind == 'node'
73+
with:
74+
node-version: 22
75+
76+
- name: Enable Corepack
77+
if: matrix.kind == 'node'
78+
run: corepack enable
79+
80+
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7
81+
if: matrix.kind == 'rust'
82+
6383
- uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7
84+
if: matrix.kind == 'node'
85+
with:
86+
targets: wasm32-unknown-unknown
6487

6588
- uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
6689
with:
6790
key: bench-${{ matrix.cache_key }}
6891

92+
- name: Install JS dependencies
93+
if: matrix.kind == 'node'
94+
run: corepack pnpm install --ignore-scripts --no-frozen-lockfile
95+
96+
- name: Install wasm-pack
97+
if: matrix.kind == 'node'
98+
shell: bash
99+
run: |
100+
set -euo pipefail
101+
102+
install_script='https://rustwasm.github.io/wasm-pack/installer/init.sh'
103+
104+
for attempt in 1 2 3; do
105+
if curl "$install_script" -sSf | sh; then
106+
exit 0
107+
fi
108+
109+
echo "wasm-pack installer attempt $attempt failed; retrying..." >&2
110+
sleep 5
111+
done
112+
113+
cargo install wasm-pack --locked --version 0.13.1
114+
115+
- name: Build JS benchmark packages
116+
if: matrix.kind == 'node'
117+
run: |
118+
corepack pnpm --filter @srcmap/codec build
119+
corepack pnpm --filter @srcmap/sourcemap build
120+
corepack pnpm --filter @srcmap/sourcemap-wasm build
121+
69122
- name: Download real-world fixtures
70123
if: matrix.fixtures
71124
run: node benchmarks/download-fixtures.mjs
72125

73126
- name: Install cargo-codspeed
127+
if: matrix.kind == 'rust'
74128
uses: taiki-e/install-action@15449e3094499af05d8d964a1c884208e4b8b595 # v2.81.11
75129
with:
76130
tool: cargo-codspeed@4.7.0
77131

78132
- name: Build benchmark shard
79-
run: cargo codspeed build -p ${{ matrix.package }} --bench ${{ matrix.bench }} --features codspeed
133+
if: matrix.kind == 'rust'
134+
run: cargo codspeed build -p ${{ matrix.package }} --bench ${{ matrix.bench }} --features ${{ matrix.features }}
80135

81136
- name: Run benchmark shard
137+
if: matrix.kind == 'rust'
82138
uses: CodSpeedHQ/action@c145068895e045cc725ee76fcd2307624b65c3af # v4.17.5
83139
with:
84-
mode: simulation
140+
mode: ${{ matrix.mode }}
85141
run: cargo codspeed run -p ${{ matrix.package }} --bench ${{ matrix.bench }}
142+
143+
- name: Run package benchmark shard
144+
if: matrix.kind == 'node'
145+
uses: CodSpeedHQ/action@c145068895e045cc725ee76fcd2307624b65c3af # v4.17.5
146+
with:
147+
mode: ${{ matrix.mode }}
148+
run: ${{ matrix.command }}

0 commit comments

Comments
 (0)