fix: make benchmark workloads deterministic

Author: BartWaardenburg

benchmarks/package.json

@@ -4,6 +4,7 @@
   "private": true,
   "type": "module",
   "scripts": {
+    "test": "node --test workload.test.mjs",
     "bench:codec": "node codec.mjs",
     "bench:sourcemap": "node sourcemap.mjs",
     "bench:wasm": "node sourcemap-wasm.mjs",

benchmarks/real-world.mjs

@@ -1,4 +1,5 @@
 import { createBench, latencyMeanMs, latencyP99Ms, throughputHz } from "./codspeed.mjs";
+import { createDeterministicLookups, setFailureExitCode } from "./workload.mjs";
 import { readFileSync, existsSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -13,6 +14,9 @@ import { SourceMap as NapiSourceMap } from "../packages/sourcemap/index.js";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const fixturesDir = join(__dirname, "fixtures");
+const LOOKUP_COUNT = 1_000;
+const LOOKUP_MAX_COLUMN = 200;
+const LOOKUP_SEED = 0x5eed1234;
 
 // ── Load fixtures ────────────────────────────────────────────────
 
@@ -80,6 +84,8 @@ const normalizePath = (s) => s?.replace(/\/\.\//g, "/") ?? null;
 
 console.log("\n--- Correctness Check ---\n");
 
+const correctnessResults = [];
+
 for (const { name, json } of maps) {
   const trace = new TraceMap(json);
   const wasm = new SourceMap(json);
@@ -130,8 +136,11 @@ for (const { name, json } of maps) {
   console.log(
     `  ${name}: WASM ${wasmPass ? "PASS" : "FAIL"}, NAPI ${napiPass ? "PASS" : "FAIL"} (${checked} lookups)`,
   );
+  correctnessResults.push({ wasmPass, napiPass });
 }
 
+setFailureExitCode(correctnessResults);
+
 // ── Parse benchmarks ─────────────────────────────────────────────
 
 console.log("\n--- Parse ---\n");
@@ -224,15 +233,8 @@ for (const { name, json, size } of maps) {
   const napi = new NapiSourceMap(json);
   const maxLine = wasm.lineCount;
 
-  const lookups = [];
-  const flatPositions = [];
-
-  for (let i = 0; i < 1000; i++) {
-    const line = Math.floor(Math.random() * maxLine);
-    const column = Math.floor(Math.random() * 200);
-    lookups.push({ line, column });
-    flatPositions.push(line, column);
-  }
+  const lookups = createDeterministicLookups(LOOKUP_COUNT, maxLine, LOOKUP_MAX_COLUMN, LOOKUP_SEED);
+  const flatPositions = lookups.flatMap(({ line, column }) => [line, column]);
   const posArray = new Int32Array(flatPositions);
 
   const isLargeMap = size > 1024 * 1024;

benchmarks/workload.mjs

@@ -0,0 +1,38 @@
+const FAILURE_EXIT_CODE = 1;
+const UINT32_RANGE = 4_294_967_296;
+
+const createRandom = (seed) => {
+  let state = seed >>> 0;
+
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let value = state;
+    value = Math.imul(value ^ (value >>> 15), value | 1);
+    value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+    return ((value ^ (value >>> 14)) >>> 0) / UINT32_RANGE;
+  };
+};
+
+/** Create repeatable zero-based lookup positions within exclusive bounds. */
+export const createDeterministicLookups = (count, maxLine, maxColumn, seed) => {
+  const random = createRandom(seed);
+  const lookups = [];
+
+  for (let index = 0; index < count; index++) {
+    lookups.push({
+      line: Math.floor(random() * maxLine),
+      column: Math.floor(random() * maxColumn),
+    });
+  }
+
+  return lookups;
+};
+
+/** Mark the benchmark process as failed when any implementation mismatches. */
+export const setFailureExitCode = (results) => {
+  const failed = results.some(({ wasmPass, napiPass }) => !wasmPass || !napiPass);
+
+  if (failed) {
+    process.exitCode = FAILURE_EXIT_CODE;
+  }
+};

benchmarks/workload.test.mjs

@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import { afterEach, test } from "node:test";
+
+import { createDeterministicLookups, setFailureExitCode } from "./workload.mjs";
+
+afterEach(() => {
+  process.exitCode = undefined;
+});
+
+test("identical seeds create identical lookups", () => {
+  const first = createDeterministicLookups(100, 80, 200, 12345);
+  const second = createDeterministicLookups(100, 80, 200, 12345);
+
+  assert.deepEqual(first, second);
+});
+
+test("different seeds create different lookups", () => {
+  const first = createDeterministicLookups(100, 80, 200, 12345);
+  const second = createDeterministicLookups(100, 80, 200, 54321);
+
+  assert.notDeepEqual(first, second);
+});
+
+test("lookups stay within the configured bounds", () => {
+  const lookups = createDeterministicLookups(1_000, 80, 200, 12345);
+
+  assert.equal(lookups.length, 1_000);
+  for (const { line, column } of lookups) {
+    assert.ok(line >= 0 && line < 80);
+    assert.ok(column >= 0 && column < 200);
+  }
+});
+
+test("a failed WASM correctness result sets the process exit code", () => {
+  setFailureExitCode([
+    { wasmPass: true, napiPass: true },
+    { wasmPass: false, napiPass: true },
+  ]);
+
+  assert.equal(process.exitCode, 1);
+});
+
+test("a failed NAPI correctness result sets the process exit code", () => {
+  setFailureExitCode([
+    { wasmPass: true, napiPass: true },
+    { wasmPass: true, napiPass: false },
+  ]);
+
+  assert.equal(process.exitCode, 1);
+});
+
+test("passing correctness results leave the process exit code unchanged", () => {
+  setFailureExitCode([{ wasmPass: true, napiPass: true }]);
+
+  assert.equal(process.exitCode, undefined);
+});

package.json

@@ -29,7 +29,7 @@
     "typos": "typos",
     "deny": "cargo deny check",
     "test": "pnpm run test:rust && pnpm run test:js",
-    "test:js": "node --test .github/scripts/check-napi-declarations.test.mjs .github/scripts/publish-crate-if-needed.test.mjs .github/scripts/workflow-policy.test.mjs packages/codec/__tests__/codec.test.mjs packages/sourcemap/__tests__/sourcemap.test.mjs packages/sourcemap-wasm/__tests__/sourcemap-wasm.test.mjs packages/sourcemap-wasm/__tests__/coverage-utils.test.mjs packages/sourcemap-wasm/__tests__/browser.test.mjs packages/generator-wasm/__tests__/generator-wasm.test.mjs packages/remapping-wasm/__tests__/remapping-wasm.test.mjs packages/trace-mapping/__tests__/trace-mapping.test.mjs packages/trace-mapping/__tests__/compat.test.mjs packages/source-map/__tests__/source-map.test.mjs packages/gen-mapping/__tests__/gen-mapping.test.mjs packages/gen-mapping/__tests__/gen-mapping.cjs.test.cjs packages/remapping/__tests__/remapping.test.mjs packages/remapping/__tests__/remapping.cjs.test.cjs packages/remapping/__tests__/compat.test.mjs",
+    "test:js": "node --test .github/scripts/check-napi-declarations.test.mjs .github/scripts/publish-crate-if-needed.test.mjs .github/scripts/workflow-policy.test.mjs benchmarks/workload.test.mjs packages/codec/__tests__/codec.test.mjs packages/sourcemap/__tests__/sourcemap.test.mjs packages/sourcemap-wasm/__tests__/sourcemap-wasm.test.mjs packages/sourcemap-wasm/__tests__/coverage-utils.test.mjs packages/sourcemap-wasm/__tests__/browser.test.mjs packages/generator-wasm/__tests__/generator-wasm.test.mjs packages/remapping-wasm/__tests__/remapping-wasm.test.mjs packages/trace-mapping/__tests__/trace-mapping.test.mjs packages/trace-mapping/__tests__/compat.test.mjs packages/source-map/__tests__/source-map.test.mjs packages/gen-mapping/__tests__/gen-mapping.test.mjs packages/gen-mapping/__tests__/gen-mapping.cjs.test.cjs packages/remapping/__tests__/remapping.test.mjs packages/remapping/__tests__/remapping.cjs.test.cjs packages/remapping/__tests__/compat.test.mjs",
     "test:rust": "cargo test",
     "coverage": "pnpm run coverage:rust && pnpm run coverage:js",
     "coverage:js": "mkdir -p coverage && node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=coverage/js-lcov.info --test-reporter=spec --test-reporter-destination=stdout packages/codec/__tests__/codec.test.mjs packages/sourcemap/__tests__/sourcemap.test.mjs packages/sourcemap-wasm/__tests__/sourcemap-wasm.test.mjs packages/sourcemap-wasm/__tests__/coverage-utils.test.mjs packages/generator-wasm/__tests__/generator-wasm.test.mjs packages/remapping-wasm/__tests__/remapping-wasm.test.mjs packages/trace-mapping/__tests__/trace-mapping.test.mjs packages/trace-mapping/__tests__/compat.test.mjs packages/source-map/__tests__/source-map.test.mjs packages/gen-mapping/__tests__/gen-mapping.test.mjs packages/gen-mapping/__tests__/gen-mapping.cjs.test.cjs packages/remapping/__tests__/remapping.test.mjs packages/remapping/__tests__/remapping.cjs.test.cjs packages/remapping/__tests__/compat.test.mjs",