benchmark.ts

#!/usr/bin/env node

/**
 * Benchmark runner — measures codegraph performance on itself (dogfooding).
 *
 * Each engine (native / WASM) runs in a forked subprocess so that a segfault
 * in the native addon only kills the child — the parent survives and collects
 * partial results from whichever engines succeeded.
 *
 * Usage: node scripts/benchmark.js
 */

import fs from 'node:fs';
import path from 'node:path';
import { performance } from 'node:perf_hooks';
import { fileURLToPath } from 'node:url';
import Database from 'better-sqlite3';
import { resolveBenchmarkExcludes, resolveBenchmarkSource, srcImport } from './lib/bench-config.js';
import { isWorker, workerEngine, workerTargets, forkEngines } from './lib/fork-engine.js';

// ── Parent process: fork one child per engine, assemble final output ─────
if (!isWorker()) {
	const { version, cleanup: versionCleanup } = await resolveBenchmarkSource();
	let wasm, native;
	try {
		({ wasm, native } = await forkEngines(import.meta.url, process.argv.slice(2)));
	} catch (err) {
		console.error(`Error: ${err.message}`);
		versionCleanup();
		process.exit(1);
	}

	const primary = wasm || native;
	if (!primary) {
		console.error('Error: Both engines failed. No results to report.');
		versionCleanup();
		process.exit(1);
	}

	function formatEngineResult(data) {
		if (!data) return null;
		return {
			files: data.files,
			buildTimeMs: data.buildTimeMs,
			queryTimeMs: data.queryTimeMs,
			nodes: data.nodes,
			edges: data.edges,
			dbSizeBytes: data.dbSizeBytes,
			perFile: data.perFile,
			noopRebuildMs: data.noopRebuildMs,
			oneFileRebuildMs: data.oneFileRebuildMs,
			oneFilePhases: data.oneFilePhases,
			queries: data.queries,
			phases: data.phases,
		};
	}

	const result = {
		version,
		date: new Date().toISOString().slice(0, 10),
		files: primary.files,
		wasm: formatEngineResult(wasm),
		native: formatEngineResult(native),
	};

	console.log(JSON.stringify(result, null, 2));
	versionCleanup();
	process.exit(0);
}

// ── Worker process: benchmark a single engine, write JSON to stdout ──────
const engine = workerEngine();

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');

const { srcDir, cleanup } = await resolveBenchmarkSource();

const dbPath = path.join(root, '.codegraph', 'graph.db');

const { buildGraph } = await import(srcImport(srcDir, 'domain/graph/builder.js'));
const { fnDepsData, fnImpactData, pathData, rolesData, statsData } = await import(
	srcImport(srcDir, 'domain/queries.js')
);
// v3.9.5+ parses WASM in a worker_thread that keeps the event loop alive until
// disposed. Older releases don't export disposeParsers — fall back to a no-op.
let disposeParsers = async () => {};
try {
	const parser = await import(srcImport(srcDir, 'domain/parser.js'));
	if (typeof parser.disposeParsers === 'function') disposeParsers = parser.disposeParsers;
} catch { /* older release — no worker pool to dispose */ }

const WARMUP_RUNS = 2;
const INCREMENTAL_RUNS = 5;
const QUERY_RUNS = 5;
const QUERY_WARMUP_RUNS = 3;
const PROBE_FILE = path.join(root, 'src', 'domain', 'queries.ts');
const BENCH_EXCLUDE = [...resolveBenchmarkExcludes()];

function median(arr) {
	const sorted = [...arr].sort((a, b) => a - b);
	const mid = Math.floor(sorted.length / 2);
	return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
}

function round1(n) {
	return Math.round(n * 10) / 10;
}

function selectTargets() {
	const db = new Database(dbPath, { readonly: true });
	const rows = db
		.prepare(
			`SELECT n.name, COUNT(e.id) AS cnt
       FROM nodes n
       JOIN edges e ON e.source_id = n.id OR e.target_id = n.id
       WHERE n.file NOT LIKE '%test%' AND n.file NOT LIKE '%spec%'
       GROUP BY n.id
       ORDER BY cnt DESC`,
		)
		.all();
	db.close();

	if (rows.length === 0) return { hub: 'buildGraph', leaf: 'median' };
	return { hub: rows[0].name, leaf: rows[rows.length - 1].name };
}

// Redirect console.log to stderr so only JSON goes to stdout
const origLog = console.log;
console.log = (...args) => console.error(...args);

// Clean DB for a full build
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);

const buildStart = performance.now();
const buildResult = await buildGraph(root, { engine, incremental: false, exclude: BENCH_EXCLUDE });
const buildTimeMs = performance.now() - buildStart;

// Warmed median of QUERY_RUNS samples with `noTests: true` to match the
// methodology used by query-benchmark.ts and the per-target `queries.*Ms`
// block below (which calls `benchQuery`, also warmed). Earlier versions of
// this script measured a single cold call, which conflated steady-state
// query latency with NAPI/rusqlite/OS-page-cache init costs (~65ms on
// macOS) and inflated growth from test-fixture files pulled in by new
// native extractors. See #1113 for the methodology rationale.
const queryTimeMs = benchQuery(fnDepsData, 'buildGraph', dbPath, { depth: 3, noTests: true });

const stats = statsData(dbPath);
const totalFiles = stats.files.total;
const totalNodes = stats.nodes.total;
const totalEdges = stats.edges.total;
const dbSizeBytes = fs.statSync(dbPath).size;

// ── Incremental build tiers ─────────────────────────────────────────
console.error(`  [${engine}] Benchmarking no-op rebuild...`);
let noopRebuildMs = null;
try {
	for (let i = 0; i < WARMUP_RUNS; i++) {
		await buildGraph(root, { engine, incremental: true, exclude: BENCH_EXCLUDE });
	}
	const noopTimings = [];
	for (let i = 0; i < INCREMENTAL_RUNS; i++) {
		const start = performance.now();
		await buildGraph(root, { engine, incremental: true, exclude: BENCH_EXCLUDE });
		noopTimings.push(performance.now() - start);
	}
	noopRebuildMs = Math.round(median(noopTimings));
} catch (err) {
	console.error(`  [${engine}] No-op rebuild failed: ${(err as Error).message}`);
}

console.error(`  [${engine}] Benchmarking 1-file rebuild...`);
const original = fs.readFileSync(PROBE_FILE, 'utf8');
let oneFileRebuildMs = null;
let oneFilePhases = null;
try {
	for (let i = 0; i < WARMUP_RUNS; i++) {
		fs.writeFileSync(PROBE_FILE, original + `\n// warmup-${i}\n`);
		await buildGraph(root, { engine, incremental: true, exclude: BENCH_EXCLUDE });
	}
	const oneFileRuns = [];
	for (let i = 0; i < INCREMENTAL_RUNS; i++) {
		fs.writeFileSync(PROBE_FILE, original + `\n// probe-${i}\n`);
		const start = performance.now();
		const res = await buildGraph(root, { engine, incremental: true, exclude: BENCH_EXCLUDE });
		oneFileRuns.push({ ms: performance.now() - start, phases: res?.phases || null });
	}
	oneFileRuns.sort((a, b) => a.ms - b.ms);
	const medianRun = oneFileRuns[Math.floor(oneFileRuns.length / 2)];
	oneFileRebuildMs = Math.round(medianRun.ms);
	oneFilePhases = medianRun.phases;
} catch (err) {
	console.error(`  [${engine}] 1-file rebuild failed: ${(err as Error).message}`);
} finally {
	fs.writeFileSync(PROBE_FILE, original);
	try {
		await buildGraph(root, { engine, incremental: true, exclude: BENCH_EXCLUDE });
	} catch {
		// Cleanup rebuild failed — probe file is already restored, move on
	}
}

// ── Query benchmarks ────────────────────────────────────────────────
console.error(`  [${engine}] Benchmarking queries...`);
const targets = workerTargets() || selectTargets();
console.error(`    hub=${targets.hub}, leaf=${targets.leaf}`);

function benchQuery(fn, ...args) {
	// Warmup runs prime NAPI bindings, the rusqlite statement cache, and the
	// OS page cache so the timed loop measures steady-state query latency
	// rather than first-call init (~65ms on macOS). Each call site warms
	// independently — methodology does not rely on call ordering elsewhere.
	for (let i = 0; i < QUERY_WARMUP_RUNS; i++) fn(...args);
	const timings = [];
	for (let i = 0; i < QUERY_RUNS; i++) {
		const start = performance.now();
		fn(...args);
		timings.push(performance.now() - start);
	}
	return round1(median(timings));
}

const queries = {
	fnDepsMs: fnDepsData ? benchQuery(fnDepsData, targets.hub, dbPath, { depth: 3, noTests: true }) : null,
	fnImpactMs: fnImpactData ? benchQuery(fnImpactData, targets.hub, dbPath, { depth: 3, noTests: true }) : null,
	pathMs: pathData ? benchQuery(pathData, targets.hub, targets.leaf, dbPath, { noTests: true }) : null,
	rolesMs: rolesData ? benchQuery(rolesData, dbPath, { noTests: true }) : null,
};

// Restore console.log for JSON output
console.log = origLog;

const workerResult = {
	buildTimeMs: Math.round(buildTimeMs),
	queryTimeMs: Math.round(queryTimeMs * 10) / 10,
	nodes: totalNodes,
	edges: totalEdges,
	files: totalFiles,
	dbSizeBytes,
	perFile: {
		buildTimeMs: Math.round((buildTimeMs / totalFiles) * 10) / 10,
		nodes: Math.round((totalNodes / totalFiles) * 10) / 10,
		edges: Math.round((totalEdges / totalFiles) * 10) / 10,
		dbSizeBytes: Math.round(dbSizeBytes / totalFiles),
	},
	noopRebuildMs,
	oneFileRebuildMs,
	oneFilePhases,
	queries,
	targets,
	phases: buildResult?.phases || null,
};

console.log(JSON.stringify(workerResult));

await disposeParsers();
cleanup();
process.exit(0);