From 494d44811affbcedc6c807e4637b00232b1e2279 Mon Sep 17 00:00:00 2001 From: Yaacov Rydzinski Date: Thu, 23 Apr 2026 05:58:10 +0300 Subject: [PATCH] feat(benchmark): use mitata for benchmark timing --- benchmark/GraphQLSchema-benchmark.js | 6 +--- benchmark/buildASTSchema-benchmark.js | 5 +-- benchmark/buildClientSchema-benchmark.js | 6 ++-- .../introspectionFromSchema-benchmark.js | 5 +-- benchmark/list-async-benchmark.js | 8 ++--- benchmark/list-asyncIterable-benchmark.js | 8 ++--- benchmark/list-sync-benchmark.js | 8 ++--- benchmark/parser-benchmark.js | 5 +-- benchmark/printer-benchmark.js | 5 +-- benchmark/repeated-fields-benchmark.js | 5 +-- benchmark/validateGQL-benchmark.js | 5 +-- benchmark/validateInvalidGQL-benchmark.js | 5 +-- benchmark/validateSDL-benchmark.js | 5 +-- benchmark/visit-benchmark.js | 5 +-- benchmark/visitInParallel-benchmark.js | 5 +-- cspell.yml | 1 + package-lock.json | 8 +++++ package.json | 1 + resources/benchmark/config.ts | 3 +- resources/benchmark/sampling.ts | 25 ++------------ resources/benchmark/statistics.ts | 8 ++--- resources/benchmark/types.ts | 5 --- resources/benchmark/worker-memory.js | 31 +++++++++-------- resources/benchmark/worker-timing.js | 34 ++++--------------- resources/benchmark/workers.ts | 11 +++--- 25 files changed, 72 insertions(+), 141 deletions(-) diff --git a/benchmark/GraphQLSchema-benchmark.js b/benchmark/GraphQLSchema-benchmark.js index f9ee9eee54..d1e1286210 100644 --- a/benchmark/GraphQLSchema-benchmark.js +++ b/benchmark/GraphQLSchema-benchmark.js @@ -7,9 +7,5 @@ const bigSchema = buildClientSchema(bigSchemaIntrospectionResult.data); export const benchmark = { name: 'Recreate a GraphQLSchema', - count: 40, - measure() { - // eslint-disable-next-line no-new - new GraphQLSchema(bigSchema.toConfig()); - }, + measure: () => new GraphQLSchema(bigSchema.toConfig()), }; diff --git a/benchmark/buildASTSchema-benchmark.js b/benchmark/buildASTSchema-benchmark.js index 10e0bcec33..46f8e432dd 100644 --- a/benchmark/buildASTSchema-benchmark.js +++ b/benchmark/buildASTSchema-benchmark.js @@ -7,8 +7,5 @@ const schemaAST = parse(bigSchemaSDL); export const benchmark = { name: 'Build Schema from AST', - count: 10, - measure() { - buildASTSchema(schemaAST, { assumeValid: true }); - }, + measure: () => buildASTSchema(schemaAST, { assumeValid: true }), }; diff --git a/benchmark/buildClientSchema-benchmark.js b/benchmark/buildClientSchema-benchmark.js index a3e8723726..13f8cca1ac 100644 --- a/benchmark/buildClientSchema-benchmark.js +++ b/benchmark/buildClientSchema-benchmark.js @@ -4,8 +4,6 @@ import { bigSchemaIntrospectionResult } from './fixtures.js'; export const benchmark = { name: 'Build Schema from Introspection', - count: 10, - measure() { - buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true }); - }, + measure: () => + buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true }), }; diff --git a/benchmark/introspectionFromSchema-benchmark.js b/benchmark/introspectionFromSchema-benchmark.js index 7e77b36f19..e0b317a302 100644 --- a/benchmark/introspectionFromSchema-benchmark.js +++ b/benchmark/introspectionFromSchema-benchmark.js @@ -10,8 +10,5 @@ const document = parse(getIntrospectionQuery()); export const benchmark = { name: 'Execute Introspection Query', - count: 20, - measure() { - executeSync({ schema, document }); - }, + measure: () => executeSync({ schema, document }), }; diff --git a/benchmark/list-async-benchmark.js b/benchmark/list-async-benchmark.js index ed22483512..7631e889f1 100644 --- a/benchmark/list-async-benchmark.js +++ b/benchmark/list-async-benchmark.js @@ -15,12 +15,10 @@ function listField() { export const benchmark = { name: 'Execute Asynchronous List Field', - count: 10, - async measure() { - await execute({ + measure: () => + execute({ schema, document, rootValue: { listField }, - }); - }, + }), }; diff --git a/benchmark/list-asyncIterable-benchmark.js b/benchmark/list-asyncIterable-benchmark.js index 6b31ea5ef3..3880a85829 100644 --- a/benchmark/list-asyncIterable-benchmark.js +++ b/benchmark/list-asyncIterable-benchmark.js @@ -13,12 +13,10 @@ async function* listField() { export const benchmark = { name: 'Execute Async Iterable List Field', - count: 10, - async measure() { - await execute({ + measure: () => + execute({ schema, document, rootValue: { listField }, - }); - }, + }), }; diff --git a/benchmark/list-sync-benchmark.js b/benchmark/list-sync-benchmark.js index e797802aa2..ed97b02514 100644 --- a/benchmark/list-sync-benchmark.js +++ b/benchmark/list-sync-benchmark.js @@ -15,12 +15,10 @@ function listField() { export const benchmark = { name: 'Execute Synchronous List Field', - count: 10, - async measure() { - await execute({ + measure: () => + execute({ schema, document, rootValue: { listField }, - }); - }, + }), }; diff --git a/benchmark/parser-benchmark.js b/benchmark/parser-benchmark.js index 5a2bfd39da..2acfffa163 100644 --- a/benchmark/parser-benchmark.js +++ b/benchmark/parser-benchmark.js @@ -5,8 +5,5 @@ const introspectionQuery = getIntrospectionQuery(); export const benchmark = { name: 'Parse introspection query', - count: 1000, - measure() { - parse(introspectionQuery); - }, + measure: () => parse(introspectionQuery), }; diff --git a/benchmark/printer-benchmark.js b/benchmark/printer-benchmark.js index e8da1f2b97..eabf80619c 100644 --- a/benchmark/printer-benchmark.js +++ b/benchmark/printer-benchmark.js @@ -7,8 +7,5 @@ const document = parse(bigDocumentSDL); export const benchmark = { name: 'Print kitchen sink document', - count: 1000, - measure() { - print(document); - }, + measure: () => print(document), }; diff --git a/benchmark/repeated-fields-benchmark.js b/benchmark/repeated-fields-benchmark.js index b416986214..e7d5b995ef 100644 --- a/benchmark/repeated-fields-benchmark.js +++ b/benchmark/repeated-fields-benchmark.js @@ -6,8 +6,5 @@ const source = `{ ${'hello '.repeat(250)}}`; export const benchmark = { name: 'Many repeated fields', - count: 5, - measure() { - graphqlSync({ schema, source }); - }, + measure: () => graphqlSync({ schema, source }), }; diff --git a/benchmark/validateGQL-benchmark.js b/benchmark/validateGQL-benchmark.js index 6bce5032c8..36430f2903 100644 --- a/benchmark/validateGQL-benchmark.js +++ b/benchmark/validateGQL-benchmark.js @@ -10,8 +10,5 @@ const queryAST = parse(getIntrospectionQuery()); export const benchmark = { name: 'Validate Introspection Query', - count: 50, - measure() { - validate(schema, queryAST); - }, + measure: () => validate(schema, queryAST), }; diff --git a/benchmark/validateInvalidGQL-benchmark.js b/benchmark/validateInvalidGQL-benchmark.js index 68666c9211..fcb1bf86d6 100644 --- a/benchmark/validateInvalidGQL-benchmark.js +++ b/benchmark/validateInvalidGQL-benchmark.js @@ -21,8 +21,5 @@ const queryAST = parse(` export const benchmark = { name: 'Validate Invalid Query', - count: 50, - measure() { - validate(schema, queryAST); - }, + measure: () => validate(schema, queryAST), }; diff --git a/benchmark/validateSDL-benchmark.js b/benchmark/validateSDL-benchmark.js index 75466ee83d..e05589c903 100644 --- a/benchmark/validateSDL-benchmark.js +++ b/benchmark/validateSDL-benchmark.js @@ -7,8 +7,5 @@ const sdlAST = parse(bigSchemaSDL); export const benchmark = { name: 'Validate SDL Document', - count: 10, - measure() { - validateSDL(sdlAST); - }, + measure: () => validateSDL(sdlAST), }; diff --git a/benchmark/visit-benchmark.js b/benchmark/visit-benchmark.js index 528f7c6981..378136fdc7 100644 --- a/benchmark/visit-benchmark.js +++ b/benchmark/visit-benchmark.js @@ -16,8 +16,5 @@ const visitor = { export const benchmark = { name: 'Visit all AST nodes', - count: 10, - measure() { - visit(documentAST, visitor); - }, + measure: () => visit(documentAST, visitor), }; diff --git a/benchmark/visitInParallel-benchmark.js b/benchmark/visitInParallel-benchmark.js index dc4cfb42ca..9f735d8148 100644 --- a/benchmark/visitInParallel-benchmark.js +++ b/benchmark/visitInParallel-benchmark.js @@ -16,8 +16,5 @@ const visitors = new Array(50).fill({ export const benchmark = { name: 'Visit all AST nodes in parallel', - count: 10, - measure() { - visit(documentAST, visitInParallel(visitors)); - }, + measure: () => visit(documentAST, visitInParallel(visitors)), }; diff --git a/cspell.yml b/cspell.yml index 68e3e08ba8..771094672b 100644 --- a/cspell.yml +++ b/cspell.yml @@ -60,6 +60,7 @@ words: - denoland - hashbang - vitest + - mitata - Rollup - Rspack - Rsbuild diff --git a/package-lock.json b/package-lock.json index 56dc1f1166..f3993c417f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "eslint-plugin-simple-import-sort": "12.1.1", "husky": "9.1.7", "lint-staged": "16.2.7", + "mitata": "1.0.34", "mocha": "11.7.5", "prettier": "3.8.1", "ts-node": "10.9.2", @@ -5435,6 +5436,13 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mitata": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/mitata/-/mitata-1.0.34.tgz", + "integrity": "sha512-Mc3zrtNBKIMeHSCQ0XqRLo1vbdIx1wvFV9c8NJAiyho6AjNfMY8bVhbS12bwciUdd1t4rj8099CH3N3NFahaUA==", + "dev": true, + "license": "MIT" + }, "node_modules/mocha": { "version": "11.7.5", "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.5.tgz", diff --git a/package.json b/package.json index fccbc4625b..d597f7e25c 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "eslint-plugin-simple-import-sort": "12.1.1", "husky": "9.1.7", "lint-staged": "16.2.7", + "mitata": "1.0.34", "mocha": "11.7.5", "prettier": "3.8.1", "ts-node": "10.9.2", diff --git a/resources/benchmark/config.ts b/resources/benchmark/config.ts index c2cc7e6445..5dc70cf2af 100644 --- a/resources/benchmark/config.ts +++ b/resources/benchmark/config.ts @@ -8,11 +8,12 @@ export const minSamples = 5; export const memorySamplesPerBenchmark = 10; +export const timingBenchmarkNodeFlags: ReadonlyArray = ['--expose-gc']; + export const memoryBenchmarkNodeFlags: ReadonlyArray = [ '--predictable', '--no-concurrent-sweeping', '--no-minor-gc-task', '--min-semi-space-size=1280', // 1.25GB '--max-semi-space-size=1280', // 1.25GB - '--trace-gc', // no gc calls should happen during benchmark, so trace them ]; diff --git a/resources/benchmark/sampling.ts b/resources/benchmark/sampling.ts index 3d9b3a651c..6cac6017cb 100644 --- a/resources/benchmark/sampling.ts +++ b/resources/benchmark/sampling.ts @@ -1,36 +1,17 @@ import assert from 'node:assert'; import { maxTime, memorySamplesPerBenchmark, minSamples } from './config.js'; -import { yellow } from './output.js'; -import type { BenchmarkTimingSample } from './types.js'; import { sampleMemoryModule, sampleTimingModule } from './workers.js'; -export function collectTimingSamples( - modulePath: string, -): Array { - let numOfConsequentlyRejectedSamples = 0; - const samples: Array = []; +export function collectTimingSamples(modulePath: string): Array { + const samples: Array = []; // If time permits, increase sample size to reduce the margin of error. const start = Date.now(); while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) { const sample = sampleTimingModule(modulePath); - if (sample.involuntaryContextSwitches > 0) { - numOfConsequentlyRejectedSamples++; - if (numOfConsequentlyRejectedSamples === 5) { - console.error( - yellow( - ' Five or more consequent runs beings rejected because of context switching.\n' + - ' Measurement can take a significantly longer time and its correctness can also be impacted.', - ), - ); - } - continue; - } - numOfConsequentlyRejectedSamples = 0; - - assert(sample.clocked > 0); + assert(sample > 0); samples.push(sample); } return samples; diff --git a/resources/benchmark/statistics.ts b/resources/benchmark/statistics.ts index a9b2436a5f..cdaa0c6463 100644 --- a/resources/benchmark/statistics.ts +++ b/resources/benchmark/statistics.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { NS_PER_SEC } from './config.js'; -import type { BenchmarkResult, BenchmarkTimingSample } from './types.js'; +import type { BenchmarkResult } from './types.js'; // T-Distribution two-tailed critical values for 95% confidence. // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm. @@ -18,7 +18,7 @@ const tTableInfinity = 1.96; // Computes stats on benchmark results. export function computeStats( name: string, - timingSamples: ReadonlyArray, + timingSamples: ReadonlyArray, memorySamples: ReadonlyArray, ): BenchmarkResult { assert(timingSamples.length > 1); @@ -26,7 +26,7 @@ export function computeStats( // Compute the sample mean (estimate of the population mean). let mean = 0; - for (const { clocked } of timingSamples) { + for (const clocked of timingSamples) { mean += clocked; } mean /= timingSamples.length; @@ -39,7 +39,7 @@ export function computeStats( // Compute the sample variance (estimate of the population variance). let variance = 0; - for (const { clocked } of timingSamples) { + for (const clocked of timingSamples) { variance += (clocked - mean) ** 2; } variance /= timingSamples.length - 1; diff --git a/resources/benchmark/types.ts b/resources/benchmark/types.ts index 0feb4528a2..4c2b4b0ff5 100644 --- a/resources/benchmark/types.ts +++ b/resources/benchmark/types.ts @@ -3,11 +3,6 @@ export interface BenchmarkProject { projectPath: string; } -export interface BenchmarkTimingSample { - clocked: number; - involuntaryContextSwitches: number; -} - export interface BenchmarkResult { name: string; memPerOp: number; diff --git a/resources/benchmark/worker-memory.js b/resources/benchmark/worker-memory.js index b3bf06eb99..23bb33813d 100644 --- a/resources/benchmark/worker-memory.js +++ b/resources/benchmark/worker-memory.js @@ -5,25 +5,28 @@ import { writeResult, } from './worker-utils.js'; +const memoryWarmupIterations = 10; +const memorySampleIterations = 10; + runWorker(async () => { const benchmark = await loadBenchmark(readModulePath()); - await warmUp(benchmark); + writeResult(await measureMemoryUsage(benchmark)); +}); + +async function measureMemoryUsage(benchmark) { + await runIterations(benchmark, memoryWarmupIterations); const memBaseline = process.memoryUsage().heapUsed; - for (let i = 0; i < benchmark.count; ++i) { + await runIterations(benchmark, memorySampleIterations); + return ( + (process.memoryUsage().heapUsed - memBaseline) / memorySampleIterations + ); +} + +async function runIterations(benchmark, iterations) { + for (let i = 0; i < iterations; ++i) { + // Each benchmark decides whether the measurement must await async work. // eslint-disable-next-line no-await-in-loop await benchmark.measure(); } - writeResult((process.memoryUsage().heapUsed - memBaseline) / benchmark.count); -}); - -async function warmUp(benchmark) { - // It looks like 7 is a magic number to reliably trigger JIT. - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); } diff --git a/resources/benchmark/worker-timing.js b/resources/benchmark/worker-timing.js index 912db40444..854a8a7261 100644 --- a/resources/benchmark/worker-timing.js +++ b/resources/benchmark/worker-timing.js @@ -1,3 +1,7 @@ +import assert from 'node:assert'; + +import { measure } from 'mitata'; + import { loadBenchmark, readModulePath, @@ -7,32 +11,8 @@ import { runWorker(async () => { const benchmark = await loadBenchmark(readModulePath()); - await warmUp(benchmark); + assert(globalThis.gc != null); - const resourcesStart = process.resourceUsage(); - const startTime = process.hrtime.bigint(); - for (let i = 0; i < benchmark.count; ++i) { - // eslint-disable-next-line no-await-in-loop - await benchmark.measure(); - } - const timeDiff = Number(process.hrtime.bigint() - startTime); - const resourcesEnd = process.resourceUsage(); - - writeResult({ - clocked: timeDiff / benchmark.count, - involuntaryContextSwitches: - resourcesEnd.involuntaryContextSwitches - - resourcesStart.involuntaryContextSwitches, - }); + const timingStats = await measure(benchmark.measure); + writeResult(timingStats.avg); }); - -async function warmUp(benchmark) { - // It looks like 7 is a magic number to reliably trigger JIT. - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); -} diff --git a/resources/benchmark/workers.ts b/resources/benchmark/workers.ts index 9a4b2dc191..bb95dfd565 100644 --- a/resources/benchmark/workers.ts +++ b/resources/benchmark/workers.ts @@ -3,8 +3,10 @@ import childProcess from 'node:child_process'; import { localRepoPath } from '../utils.js'; -import { memoryBenchmarkNodeFlags } from './config.js'; -import type { BenchmarkTimingSample } from './types.js'; +import { + memoryBenchmarkNodeFlags, + timingBenchmarkNodeFlags, +} from './config.js'; export function getBenchmarkName(modulePath: string): string { return runWorkerFile( @@ -13,11 +15,12 @@ export function getBenchmarkName(modulePath: string): string { ) as string; } -export function sampleTimingModule(modulePath: string): BenchmarkTimingSample { +export function sampleTimingModule(modulePath: string): number { return runWorkerFile( localRepoPath('resources/benchmark/worker-timing.js'), modulePath, - ) as BenchmarkTimingSample; + timingBenchmarkNodeFlags, + ) as number; } export function sampleMemoryModule(modulePath: string): number {