fix: fail benchmark reindex when indexer exits non-zero (#117)

* fix: fail benchmark reindex when indexer exits non-zero

Check child exit code and surface stderr instead of recording timings for failed indexer runs.

* fix: validate benchmark reindex runs is a positive integer

Author: SutuSebastian

.changeset/fix-benchmark-reindex-exit-code.md

@@ -0,0 +1,5 @@
+---
+"@stainless-code/codemap": patch
+---
+
+Fail benchmark reindex runs when the spawned indexer exits non-zero instead of recording misleading timings.

src/benchmark-reindex.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "bun:test";
+
+import { runBenchmarkReindex } from "./benchmark-reindex";
+
+describe("runBenchmarkReindex", () => {
+  it("throws when the indexer exits non-zero instead of recording a timing", async () => {
+    let spawnCalls = 0;
+    await expect(
+      runBenchmarkReindex("fail case", ["--full"], {
+        runs: 1,
+        spawnIndexer: async () => {
+          spawnCalls++;
+          return {
+            exitCode: 1,
+            stderr: "indexer exploded",
+            stdout: "",
+          };
+        },
+      }),
+    ).rejects.toThrow(/benchmark reindex "fail case" failed \(exit 1\)/);
+    expect(spawnCalls).toBe(1);
+  });
+
+  it("rejects invalid runs values", async () => {
+    const spawnIndexer = async () => ({
+      exitCode: 0,
+      stderr: "",
+      stdout: "",
+    });
+    await expect(
+      runBenchmarkReindex("bad-runs", [], { runs: 0, spawnIndexer }),
+    ).rejects.toThrow(/requires runs >= 1/);
+    await expect(
+      runBenchmarkReindex("bad-runs", [], { runs: -1, spawnIndexer }),
+    ).rejects.toThrow(/requires runs >= 1/);
+    await expect(
+      runBenchmarkReindex("bad-runs", [], { runs: 1.5, spawnIndexer }),
+    ).rejects.toThrow(/requires runs >= 1/);
+  });
+
+  it("records timings when every run exits zero", async () => {
+    const result = await runBenchmarkReindex("ok", [], {
+      runs: 2,
+      spawnIndexer: async () => ({
+        exitCode: 0,
+        stderr: "",
+        stdout: "",
+      }),
+    });
+    expect(result.label).toBe("ok");
+    expect(result.runs).toBe(2);
+    expect(result.avg).toBeGreaterThanOrEqual(0);
+  });
+});

src/benchmark-reindex.ts

@@ -0,0 +1,47 @@
+async function timeMsAsync(fn: () => Promise<void>): Promise<{ ms: number }> {
+  const start = performance.now();
+  await fn();
+  return { ms: performance.now() - start };
+}
+
+export interface IndexerSpawnResult {
+  exitCode: number | null;
+  stderr: string;
+  stdout: string;
+}
+
+export type IndexerSpawn = (args: string[]) => Promise<IndexerSpawnResult>;
+
+export async function runBenchmarkReindex(
+  label: string,
+  args: string[],
+  opts: { spawnIndexer: IndexerSpawn; runs?: number },
+): Promise<{
+  label: string;
+  avg: number;
+  min: number;
+  max: number;
+  runs: number;
+}> {
+  const runs = opts.runs ?? 3;
+  if (!Number.isInteger(runs) || runs < 1) {
+    throw new Error(`benchmark reindex "${label}" requires runs >= 1`);
+  }
+  const times: number[] = [];
+  for (let i = 0; i < runs; i++) {
+    const t = await timeMsAsync(async () => {
+      const { exitCode, stderr, stdout } = await opts.spawnIndexer(args);
+      if (exitCode !== 0) {
+        const detail = [stderr, stdout].filter(Boolean).join("\n").trim();
+        throw new Error(
+          `benchmark reindex "${label}" failed (exit ${exitCode ?? "?"}): ${detail || "(no output)"}`,
+        );
+      }
+    });
+    times.push(t.ms);
+  }
+  const avg = times.reduce((a, b) => a + b, 0) / runs;
+  const min = Math.min(...times);
+  const max = Math.max(...times);
+  return { label, avg, min, max, runs };
+}

src/benchmark.ts

@@ -4,6 +4,7 @@ import { join, resolve } from "node:path";
 import { loadScenariosFromConfigFile } from "./benchmark-config";
 import { getDefaultScenarios } from "./benchmark-default-scenarios";
 import type { Scenario } from "./benchmark-default-scenarios";
+import { runBenchmarkReindex } from "./benchmark-reindex";
 import { loadUserConfig, resolveCodemapConfig } from "./config";
 import { closeDb, openDb } from "./db";
 import { configureResolver } from "./resolver";
@@ -40,14 +41,6 @@ function timeMs(fn: () => unknown): { result: unknown; ms: number } {
   return { result, ms: performance.now() - start };
 }
 
-async function timeMsAsync(
-  fn: () => Promise<unknown>,
-): Promise<{ result: unknown; ms: number }> {
-  const start = performance.now();
-  const result = await fn();
-  return { result, ms: performance.now() - start };
-}
-
 function fmtBytes(b: number): string {
   if (b < 1024) return `${b} B`;
   if (b < 1024 * 1024) return `${(b / 1024).toFixed(1)} KB`;
@@ -182,25 +175,22 @@ console.log(
 
 const INDEXER_PATH = join(import.meta.dirname, "index.ts");
 
-async function benchmarkReindex(label: string, args: string[]) {
-  const runs = 3;
-  const times: number[] = [];
-  for (let i = 0; i < runs; i++) {
-    const t = await timeMsAsync(async () => {
-      const proc = Bun.spawn(["bun", INDEXER_PATH, ...args], {
-        cwd: getProjectRoot(),
-        stdout: "pipe",
-        stderr: "pipe",
-      });
-      await proc.exited;
-      return proc.exitCode;
-    });
-    times.push(t.ms);
-  }
-  const avg = times.reduce((a, b) => a + b, 0) / runs;
-  const min = Math.min(...times);
-  const max = Math.max(...times);
-  return { label, avg, min, max, runs };
+async function spawnIndexer(args: string[]) {
+  const proc = Bun.spawn(["bun", INDEXER_PATH, ...args], {
+    cwd: getProjectRoot(),
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  await proc.exited;
+  return {
+    exitCode: proc.exitCode,
+    stderr: await new Response(proc.stderr).text(),
+    stdout: await new Response(proc.stdout).text(),
+  };
+}
+
+function benchmarkReindex(label: string, args: string[]) {
+  return runBenchmarkReindex(label, args, { spawnIndexer });
 }
 
 console.log("  ─── Reindex Benchmarks ───\n");