Skip to content

Commit b476409

Browse files
authored
fix: stabilize benchmark targets across engines and preserve README links (#527)
* fix: stabilize benchmark target selection across engines and preserve README links - Share selectTargets() results from first engine worker to second via env, ensuring both engines benchmark the same hub/mid/leaf symbols - Preserve existing README benchmark links instead of hardcoding a subset - Add regression notes explaining v3.1.4 → v3.3.0 build performance increase Impact: 2 functions changed, 4 affected * fix: use non-greedy regex to preserve markdown links in README benchmark line The previous [^)]+ pattern stopped at the first ) inside markdown link URLs, truncating and corrupting the captured link string on every run. Switch to (.+?): which anchors on the outer ): delimiter. * fix: remove dead results.native fallback in target propagation results.native is always null at this point since native runs after target propagation. The || results.native arm was unreachable. Impact: 1 functions changed, 0 affected
1 parent 202d35a commit b476409

5 files changed

Lines changed: 48 additions & 5 deletions

File tree

generated/benchmarks/BUILD-BENCHMARKS.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,18 @@ remains at 6.6 ms/file (vs 5.0 in v2.0.0). The WASM/Native ratio widened from
166166
2.0x to 3.5x. Further optimization of WASM boundary crossings in the JS
167167
extractor is needed to recover the regression.
168168

169+
**Build regression (v3.1.4 3.5 ms/file → v3.3.0 8 ms/file, +129% native):** The codebase grew from
170+
398 to 429 files (+8%), but the per-file regression is real and driven by richer extraction. Between
171+
v3.1.4 and v3.3.0, type inference was extended to all typed languages (#501), receiver type tracking
172+
with graded confidence was added (#505), re-exported barrel file symbols are now tracked (#515), and
173+
package.json exports + monorepo workspace resolution was introduced (#509). These produce 33% more
174+
nodes/file (13.4 → 17.8) and 28% more edges/file (28.8 → 36.8). The Parse phase tripled on native
175+
(468 → 1511 ms) because extractors now perform additional AST traversals for type annotations and
176+
receiver resolution. The Complexity phase grew 10× (16 → 179 ms) because 33% more functions each
177+
require full AST analysis. Major refactors also decomposed monolithic extractors into per-category
178+
handlers (#490) and split domain/feature modules (#491, #492), adding 31 new source files — the
179+
benchmark measures codegraph on itself, so more source files amplify per-file overhead.
180+
169181
**Native build regression (v3.0.0 4.4 ms/file → v3.0.3 12.3 ms/file):** The regression is entirely
170182
from new build phases added in v3.0.1 that are now default-on: AST node extraction (651ms),
171183
dataflow analysis (367ms), and CFG construction (169ms) — totalling ~1,187ms of new work. The original

scripts/benchmark.js

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { performance } from 'node:perf_hooks';
1616
import { fileURLToPath } from 'node:url';
1717
import Database from 'better-sqlite3';
1818
import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js';
19-
import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js';
19+
import { isWorker, workerEngine, workerTargets, forkEngines } from './lib/fork-engine.js';
2020

2121
// ── Parent process: fork one child per engine, assemble final output ─────
2222
if (!isWorker()) {
@@ -179,7 +179,7 @@ try {
179179

180180
// ── Query benchmarks ────────────────────────────────────────────────
181181
console.error(` [${engine}] Benchmarking queries...`);
182-
const targets = selectTargets();
182+
const targets = workerTargets() || selectTargets();
183183
console.error(` hub=${targets.hub}, leaf=${targets.leaf}`);
184184

185185
function benchQuery(fn, ...args) {
@@ -219,6 +219,7 @@ const workerResult = {
219219
oneFileRebuildMs,
220220
oneFilePhases,
221221
queries,
222+
targets,
222223
phases: buildResult?.phases || null,
223224
};
224225

scripts/lib/fork-engine.js

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import { fork } from 'node:child_process';
2525
import { fileURLToPath } from 'node:url';
2626

2727
const WORKER_ENV_KEY = '__BENCH_ENGINE__';
28+
const TARGETS_ENV_KEY = '__BENCH_TARGETS__';
2829

2930
/**
3031
* Returns true when running inside a forked worker process.
@@ -43,6 +44,16 @@ export function workerEngine() {
4344
return engine;
4445
}
4546

47+
/**
48+
* Returns pre-selected targets passed from the parent process, or null if
49+
* this is the first engine run (no targets yet).
50+
*/
51+
export function workerTargets() {
52+
const raw = process.env[TARGETS_ENV_KEY];
53+
if (!raw) return null;
54+
try { return JSON.parse(raw); } catch { return null; }
55+
}
56+
4657
/**
4758
* Fork a single worker subprocess and collect its JSON output.
4859
*
@@ -158,17 +169,28 @@ export async function forkEngines(scriptUrl, argv = [], opts = {}) {
158169
const results = { wasm: null, native: null };
159170

160171
// Run engines sequentially — they share the DB file and filesystem state.
172+
// After the first engine completes, extract its targets and pass them to
173+
// the second engine via TARGETS_ENV_KEY so both benchmark the same symbols.
161174
if (hasWasm) {
162175
results.wasm = await forkWorker(scriptPath, WORKER_ENV_KEY, 'wasm', argv, timeoutMs);
163176
} else {
164177
console.error('WASM grammars not built — skipping WASM benchmark');
165178
}
166179

180+
// Propagate targets from the first engine to the second
181+
const firstResult = results.wasm;
182+
if (firstResult?.targets) {
183+
process.env[TARGETS_ENV_KEY] = JSON.stringify(firstResult.targets);
184+
}
185+
167186
if (hasNative) {
168187
results.native = await forkWorker(scriptPath, WORKER_ENV_KEY, 'native', argv, timeoutMs);
169188
} else {
170189
console.error('Native engine not available — skipping native benchmark');
171190
}
172191

192+
// Clean up env
193+
delete process.env[TARGETS_ENV_KEY];
194+
173195
return results;
174196
}

scripts/query-benchmark.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { performance } from 'node:perf_hooks';
1717
import { fileURLToPath } from 'node:url';
1818
import Database from 'better-sqlite3';
1919
import { resolveBenchmarkSource, srcImport } from './lib/bench-config.js';
20-
import { isWorker, workerEngine, forkEngines } from './lib/fork-engine.js';
20+
import { isWorker, workerEngine, workerTargets, forkEngines } from './lib/fork-engine.js';
2121

2222
// ── Parent process: fork one child per engine, assemble final output ─────
2323
if (!isWorker()) {
@@ -186,7 +186,7 @@ function benchDiffImpact(hubName) {
186186
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
187187
await buildGraph(root, { engine, incremental: false });
188188

189-
const targets = selectTargets();
189+
const targets = workerTargets() || selectTargets();
190190
console.error(`Targets: hub=${targets.hub}, mid=${targets.mid}, leaf=${targets.leaf}`);
191191

192192
const fnDeps = {};

scripts/update-benchmark-report.js

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,9 +349,17 @@ if (fs.existsSync(readmePath)) {
349349
: formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES);
350350
rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estBuild} build** |\n`;
351351

352+
// Preserve existing benchmark link line from README rather than hardcoding.
353+
// Fall back to a default if we can't find it.
354+
let benchmarkLinks = '[build benchmarks](generated/benchmarks/BUILD-BENCHMARKS.md) | [embedding benchmarks](generated/benchmarks/EMBEDDING-BENCHMARKS.md)';
355+
const linksMatch = readme.match(/Self-measured on every release via CI \((.+?)\):/);
356+
if (linksMatch) {
357+
benchmarkLinks = linksMatch[1];
358+
}
359+
352360
const perfSection = `## 📊 Performance
353361
354-
Self-measured on every release via CI ([build benchmarks](generated/benchmarks/BUILD-BENCHMARKS.md) | [embedding benchmarks](generated/benchmarks/EMBEDDING-BENCHMARKS.md)):
362+
Self-measured on every release via CI (${benchmarkLinks}):
355363
356364
| Metric | Latest |
357365
|---|---|

0 commit comments

Comments
 (0)