internal: use mitata for benchmark timing

benchmark/GraphQLSchema-benchmark.js

@@ -7,9 +7,5 @@ const bigSchema = buildClientSchema(bigSchemaIntrospectionResult.data);
 
 export const benchmark = {
   name: 'Recreate a GraphQLSchema',
-  count: 40,
-  measure() {
-    // eslint-disable-next-line no-new
-    new GraphQLSchema(bigSchema.toConfig());
-  },
+  measure: () => new GraphQLSchema(bigSchema.toConfig()),
 };

benchmark/buildASTSchema-benchmark.js

@@ -7,8 +7,5 @@ const schemaAST = parse(bigSchemaSDL);
 
 export const benchmark = {
   name: 'Build Schema from AST',
-  count: 10,
-  measure() {
-    buildASTSchema(schemaAST, { assumeValid: true });
-  },
+  measure: () => buildASTSchema(schemaAST, { assumeValid: true }),
 };

benchmark/buildClientSchema-benchmark.js

@@ -4,8 +4,6 @@ import { bigSchemaIntrospectionResult } from './fixtures.js';
 
 export const benchmark = {
   name: 'Build Schema from Introspection',
-  count: 10,
-  measure() {
-    buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true });
-  },
+  measure: () =>
+    buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true }),
 };

benchmark/introspectionFromSchema-benchmark.js

@@ -10,8 +10,5 @@ const document = parse(getIntrospectionQuery());
 
 export const benchmark = {
   name: 'Execute Introspection Query',
-  count: 20,
-  measure() {
-    executeSync({ schema, document });
-  },
+  measure: () => executeSync({ schema, document }),
 };

benchmark/list-async-benchmark.js

@@ -15,12 +15,10 @@ function listField() {
 
 export const benchmark = {
   name: 'Execute Asynchronous List Field',
-  count: 10,
-  async measure() {
-    await execute({
+  measure: () =>
+    execute({
       schema,
       document,
       rootValue: { listField },
-    });
-  },
+    }),
 };

benchmark/list-asyncIterable-benchmark.js

@@ -13,12 +13,10 @@ async function* listField() {
 
 export const benchmark = {
   name: 'Execute Async Iterable List Field',
-  count: 10,
-  async measure() {
-    await execute({
+  measure: () =>
+    execute({
       schema,
       document,
       rootValue: { listField },
-    });
-  },
+    }),
 };

benchmark/list-sync-benchmark.js

@@ -15,12 +15,10 @@ function listField() {
 
 export const benchmark = {
   name: 'Execute Synchronous List Field',
-  count: 10,
-  async measure() {
-    await execute({
+  measure: () =>
+    execute({
       schema,
       document,
       rootValue: { listField },
-    });
-  },
+    }),
 };

benchmark/parser-benchmark.js

@@ -5,8 +5,5 @@ const introspectionQuery = getIntrospectionQuery();
 
 export const benchmark = {
   name: 'Parse introspection query',
-  count: 1000,
-  measure() {
-    parse(introspectionQuery);
-  },
+  measure: () => parse(introspectionQuery),
 };

benchmark/printer-benchmark.js

@@ -7,8 +7,5 @@ const document = parse(bigDocumentSDL);
 
 export const benchmark = {
   name: 'Print kitchen sink document',
-  count: 1000,
-  measure() {
-    print(document);
-  },
+  measure: () => print(document),
 };

benchmark/repeated-fields-benchmark.js

@@ -6,8 +6,5 @@ const source = `{ ${'hello '.repeat(250)}}`;
 
 export const benchmark = {
   name: 'Many repeated fields',
-  count: 5,
-  measure() {
-    graphqlSync({ schema, source });
-  },
+  measure: () => graphqlSync({ schema, source }),
 };

benchmark/validateGQL-benchmark.js

@@ -10,8 +10,5 @@ const queryAST = parse(getIntrospectionQuery());
 
 export const benchmark = {
   name: 'Validate Introspection Query',
-  count: 50,
-  measure() {
-    validate(schema, queryAST);
-  },
+  measure: () => validate(schema, queryAST),
 };

benchmark/validateInvalidGQL-benchmark.js

@@ -21,8 +21,5 @@ const queryAST = parse(`
 
 export const benchmark = {
   name: 'Validate Invalid Query',
-  count: 50,
-  measure() {
-    validate(schema, queryAST);
-  },
+  measure: () => validate(schema, queryAST),
 };

benchmark/validateSDL-benchmark.js

@@ -7,8 +7,5 @@ const sdlAST = parse(bigSchemaSDL);
 
 export const benchmark = {
   name: 'Validate SDL Document',
-  count: 10,
-  measure() {
-    validateSDL(sdlAST);
-  },
+  measure: () => validateSDL(sdlAST),
 };

benchmark/visit-benchmark.js

@@ -16,8 +16,5 @@ const visitor = {
 
 export const benchmark = {
   name: 'Visit all AST nodes',
-  count: 10,
-  measure() {
-    visit(documentAST, visitor);
-  },
+  measure: () => visit(documentAST, visitor),
 };

benchmark/visitInParallel-benchmark.js

@@ -16,8 +16,5 @@ const visitors = new Array(50).fill({
 
 export const benchmark = {
   name: 'Visit all AST nodes in parallel',
-  count: 10,
-  measure() {
-    visit(documentAST, visitInParallel(visitors));
-  },
+  measure: () => visit(documentAST, visitInParallel(visitors)),
 };

cspell.yml

@@ -60,6 +60,7 @@ words:
   - denoland
   - hashbang
   - vitest
+  - mitata
   - Rollup
   - Rspack
   - Rsbuild

package.json

@@ -68,6 +68,7 @@
     "eslint-plugin-simple-import-sort": "12.1.1",
     "husky": "9.1.7",
     "lint-staged": "16.2.7",
+    "mitata": "1.0.34",
     "mocha": "11.7.5",
     "prettier": "3.8.1",
     "ts-node": "10.9.2",

resources/benchmark/config.ts

@@ -8,11 +8,12 @@ export const minSamples = 5;
 
 export const memorySamplesPerBenchmark = 10;
 
+export const timingBenchmarkNodeFlags: ReadonlyArray<string> = ['--expose-gc'];
+
 export const memoryBenchmarkNodeFlags: ReadonlyArray<string> = [
   '--predictable',
   '--no-concurrent-sweeping',
   '--no-minor-gc-task',
   '--min-semi-space-size=1280', // 1.25GB
   '--max-semi-space-size=1280', // 1.25GB
-  '--trace-gc', // no gc calls should happen during benchmark, so trace them
 ];

resources/benchmark/sampling.ts

@@ -1,36 +1,17 @@
 import assert from 'node:assert';
 
 import { maxTime, memorySamplesPerBenchmark, minSamples } from './config.js';
-import { yellow } from './output.js';
-import type { BenchmarkTimingSample } from './types.js';
 import { sampleMemoryModule, sampleTimingModule } from './workers.js';
 
-export function collectTimingSamples(
-  modulePath: string,
-): Array<BenchmarkTimingSample> {
-  let numOfConsequentlyRejectedSamples = 0;
-  const samples: Array<BenchmarkTimingSample> = [];
+export function collectTimingSamples(modulePath: string): Array<number> {
+  const samples: Array<number> = [];
 
   // If time permits, increase sample size to reduce the margin of error.
   const start = Date.now();
   while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
     const sample = sampleTimingModule(modulePath);
 
-    if (sample.involuntaryContextSwitches > 0) {
-      numOfConsequentlyRejectedSamples++;
-      if (numOfConsequentlyRejectedSamples === 5) {
-        console.error(
-          yellow(
-            '  Five or more consequent runs beings rejected because of context switching.\n' +
-              '  Measurement can take a significantly longer time and its correctness can also be impacted.',
-          ),
-        );
-      }
-      continue;
-    }
-    numOfConsequentlyRejectedSamples = 0;
-
-    assert(sample.clocked > 0);
+    assert(sample > 0);
     samples.push(sample);
   }
   return samples;

resources/benchmark/statistics.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert';
 
 import { NS_PER_SEC } from './config.js';
-import type { BenchmarkResult, BenchmarkTimingSample } from './types.js';
+import type { BenchmarkResult } from './types.js';
 
 // T-Distribution two-tailed critical values for 95% confidence.
 // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
@@ -18,15 +18,15 @@ const tTableInfinity = 1.96;
 // Computes stats on benchmark results.
 export function computeStats(
   name: string,
-  timingSamples: ReadonlyArray<BenchmarkTimingSample>,
+  timingSamples: ReadonlyArray<number>,
   memorySamples: ReadonlyArray<number>,
 ): BenchmarkResult {
   assert(timingSamples.length > 1);
   assert(memorySamples.length > 0);
 
   // Compute the sample mean (estimate of the population mean).
   let mean = 0;
-  for (const { clocked } of timingSamples) {
+  for (const clocked of timingSamples) {
     mean += clocked;
   }
   mean /= timingSamples.length;
@@ -39,7 +39,7 @@ export function computeStats(
 
   // Compute the sample variance (estimate of the population variance).
   let variance = 0;
-  for (const { clocked } of timingSamples) {
+  for (const clocked of timingSamples) {
     variance += (clocked - mean) ** 2;
   }
   variance /= timingSamples.length - 1;

resources/benchmark/types.ts

@@ -3,11 +3,6 @@ export interface BenchmarkProject {
   projectPath: string;
 }
 
-export interface BenchmarkTimingSample {
-  clocked: number;
-  involuntaryContextSwitches: number;
-}
-
 export interface BenchmarkResult {
   name: string;
   memPerOp: number;

resources/benchmark/worker-memory.js

@@ -5,25 +5,28 @@ import {
   writeResult,
 } from './worker-utils.js';
 
+const memoryWarmupIterations = 10;
+const memorySampleIterations = 10;
+
 runWorker(async () => {
   const benchmark = await loadBenchmark(readModulePath());
-  await warmUp(benchmark);
+  writeResult(await measureMemoryUsage(benchmark));
+});
+
+async function measureMemoryUsage(benchmark) {
+  await runIterations(benchmark, memoryWarmupIterations);
 
   const memBaseline = process.memoryUsage().heapUsed;
-  for (let i = 0; i < benchmark.count; ++i) {
+  await runIterations(benchmark, memorySampleIterations);
+  return (
+    (process.memoryUsage().heapUsed - memBaseline) / memorySampleIterations
+  );
+}
+
+async function runIterations(benchmark, iterations) {
+  for (let i = 0; i < iterations; ++i) {
+    // Each benchmark decides whether the measurement must await async work.
     // eslint-disable-next-line no-await-in-loop
     await benchmark.measure();
   }
-  writeResult((process.memoryUsage().heapUsed - memBaseline) / benchmark.count);
-});
-
-async function warmUp(benchmark) {
-  // It looks like 7 is a magic number to reliably trigger JIT.
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
 }

resources/benchmark/worker-timing.js

@@ -1,3 +1,7 @@
+import assert from 'node:assert';
+
+import { measure } from 'mitata';
+
 import {
   loadBenchmark,
   readModulePath,
@@ -7,32 +11,8 @@ import {
 
 runWorker(async () => {
   const benchmark = await loadBenchmark(readModulePath());
-  await warmUp(benchmark);
+  assert(globalThis.gc != null);
 
-  const resourcesStart = process.resourceUsage();
-  const startTime = process.hrtime.bigint();
-  for (let i = 0; i < benchmark.count; ++i) {
-    // eslint-disable-next-line no-await-in-loop
-    await benchmark.measure();
-  }
-  const timeDiff = Number(process.hrtime.bigint() - startTime);
-  const resourcesEnd = process.resourceUsage();
-
-  writeResult({
-    clocked: timeDiff / benchmark.count,
-    involuntaryContextSwitches:
-      resourcesEnd.involuntaryContextSwitches -
-      resourcesStart.involuntaryContextSwitches,
-  });
+  const timingStats = await measure(benchmark.measure);
+  writeResult(timingStats.avg);
 });
-
-async function warmUp(benchmark) {
-  // It looks like 7 is a magic number to reliably trigger JIT.
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-  await benchmark.measure();
-}