diff --git a/resources/benchmark/config.ts b/resources/benchmark/config.ts index b241a2751b..c2cc7e6445 100644 --- a/resources/benchmark/config.ts +++ b/resources/benchmark/config.ts @@ -6,7 +6,9 @@ export const maxTime = 5; // The minimum sample size required to perform statistical analysis. export const minSamples = 5; -export const nodeFlags: ReadonlyArray = [ +export const memorySamplesPerBenchmark = 10; + +export const memoryBenchmarkNodeFlags: ReadonlyArray = [ '--predictable', '--no-concurrent-sweeping', '--no-minor-gc-task', diff --git a/resources/benchmark/run.ts b/resources/benchmark/run.ts index ce0a86b824..ec9e8cdd82 100644 --- a/resources/benchmark/run.ts +++ b/resources/benchmark/run.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { getArguments } from './args.js'; import { cyan, printBenchmarkResults, red } from './output.js'; import { prepareBenchmarkProjects } from './projects.js'; -import { collectSamples } from './sampling.js'; +import { collectMemorySamples, collectTimingSamples } from './sampling.js'; import { computeStats } from './statistics.js'; import type { BenchmarkProject, BenchmarkResult } from './types.js'; import { getBenchmarkName } from './workers.js'; @@ -33,9 +33,10 @@ function runBenchmark( } try { - const samples = collectSamples(modulePath); + const timingSamples = collectTimingSamples(modulePath); + const memorySamples = collectMemorySamples(modulePath); - results.push(computeStats(revision, samples)); + results.push(computeStats(revision, timingSamples, memorySamples)); process.stdout.write(' ' + cyan(i + 1) + ' tests completed.\u000D'); } catch (error) { const errorMessage = diff --git a/resources/benchmark/sampling.ts b/resources/benchmark/sampling.ts index deefb63849..3d9b3a651c 100644 --- a/resources/benchmark/sampling.ts +++ b/resources/benchmark/sampling.ts @@ -1,18 +1,20 @@ import assert from 'node:assert'; -import { maxTime, minSamples } from './config.js'; +import { maxTime, memorySamplesPerBenchmark, minSamples } from './config.js'; import { yellow } from './output.js'; -import type { BenchmarkSample } from './types.js'; -import { sampleModule } from './workers.js'; +import type { BenchmarkTimingSample } from './types.js'; +import { sampleMemoryModule, sampleTimingModule } from './workers.js'; -export function collectSamples(modulePath: string): Array { +export function collectTimingSamples( + modulePath: string, +): Array { let numOfConsequentlyRejectedSamples = 0; - const samples: Array = []; + const samples: Array = []; // If time permits, increase sample size to reduce the margin of error. const start = Date.now(); while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) { - const sample = sampleModule(modulePath); + const sample = sampleTimingModule(modulePath); if (sample.involuntaryContextSwitches > 0) { numOfConsequentlyRejectedSamples++; @@ -29,7 +31,20 @@ export function collectSamples(modulePath: string): Array { numOfConsequentlyRejectedSamples = 0; assert(sample.clocked > 0); - assert(sample.memUsed > 0); + samples.push(sample); + } + return samples; +} + +export function collectMemorySamples(modulePath: string): Array { + const samples: Array = []; + for ( + let sampleIndex = 0; + sampleIndex < memorySamplesPerBenchmark; + ++sampleIndex + ) { + const sample = sampleMemoryModule(modulePath); + assert(sample > 0); samples.push(sample); } return samples; diff --git a/resources/benchmark/statistics.ts b/resources/benchmark/statistics.ts index d3ea5f0289..a9b2436a5f 100644 --- a/resources/benchmark/statistics.ts +++ b/resources/benchmark/statistics.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { NS_PER_SEC } from './config.js'; -import type { BenchmarkResult, BenchmarkSample } from './types.js'; +import type { BenchmarkResult, BenchmarkTimingSample } from './types.js'; // T-Distribution two-tailed critical values for 95% confidence. // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm. @@ -18,35 +18,40 @@ const tTableInfinity = 1.96; // Computes stats on benchmark results. export function computeStats( name: string, - samples: ReadonlyArray, + timingSamples: ReadonlyArray, + memorySamples: ReadonlyArray, ): BenchmarkResult { - assert(samples.length > 1); + assert(timingSamples.length > 1); + assert(memorySamples.length > 0); // Compute the sample mean (estimate of the population mean). let mean = 0; - let meanMemUsed = 0; - for (const { clocked, memUsed } of samples) { + for (const { clocked } of timingSamples) { mean += clocked; + } + mean /= timingSamples.length; + + let meanMemUsed = 0; + for (const memUsed of memorySamples) { meanMemUsed += memUsed; } - mean /= samples.length; - meanMemUsed /= samples.length; + meanMemUsed /= memorySamples.length; // Compute the sample variance (estimate of the population variance). let variance = 0; - for (const { clocked } of samples) { + for (const { clocked } of timingSamples) { variance += (clocked - mean) ** 2; } - variance /= samples.length - 1; + variance /= timingSamples.length - 1; // Compute the sample standard deviation (estimate of the population standard deviation). const sd = Math.sqrt(variance); // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean). - const sem = sd / Math.sqrt(samples.length); + const sem = sd / Math.sqrt(timingSamples.length); // Compute the degrees of freedom. - const df = samples.length - 1; + const df = timingSamples.length - 1; // Compute the critical value. const critical = tTable[df] ?? tTableInfinity; @@ -62,6 +67,6 @@ export function computeStats( memPerOp: Math.floor(meanMemUsed), ops: NS_PER_SEC / mean, deviation: rme, - numSamples: samples.length, + numSamples: timingSamples.length, }; } diff --git a/resources/benchmark/types.ts b/resources/benchmark/types.ts index 1699c6c57f..0feb4528a2 100644 --- a/resources/benchmark/types.ts +++ b/resources/benchmark/types.ts @@ -3,9 +3,8 @@ export interface BenchmarkProject { projectPath: string; } -export interface BenchmarkSample { +export interface BenchmarkTimingSample { clocked: number; - memUsed: number; involuntaryContextSwitches: number; } diff --git a/resources/benchmark/worker-memory.js b/resources/benchmark/worker-memory.js new file mode 100644 index 0000000000..b3bf06eb99 --- /dev/null +++ b/resources/benchmark/worker-memory.js @@ -0,0 +1,29 @@ +import { + loadBenchmark, + readModulePath, + runWorker, + writeResult, +} from './worker-utils.js'; + +runWorker(async () => { + const benchmark = await loadBenchmark(readModulePath()); + await warmUp(benchmark); + + const memBaseline = process.memoryUsage().heapUsed; + for (let i = 0; i < benchmark.count; ++i) { + // eslint-disable-next-line no-await-in-loop + await benchmark.measure(); + } + writeResult((process.memoryUsage().heapUsed - memBaseline) / benchmark.count); +}); + +async function warmUp(benchmark) { + // It looks like 7 is a magic number to reliably trigger JIT. + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); +} diff --git a/resources/benchmark/worker-timing.js b/resources/benchmark/worker-timing.js index dd165a65b5..912db40444 100644 --- a/resources/benchmark/worker-timing.js +++ b/resources/benchmark/worker-timing.js @@ -9,8 +9,6 @@ runWorker(async () => { const benchmark = await loadBenchmark(readModulePath()); await warmUp(benchmark); - const memBaseline = process.memoryUsage().heapUsed; - const resourcesStart = process.resourceUsage(); const startTime = process.hrtime.bigint(); for (let i = 0; i < benchmark.count; ++i) { @@ -22,7 +20,6 @@ runWorker(async () => { writeResult({ clocked: timeDiff / benchmark.count, - memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count, involuntaryContextSwitches: resourcesEnd.involuntaryContextSwitches - resourcesStart.involuntaryContextSwitches, diff --git a/resources/benchmark/workers.ts b/resources/benchmark/workers.ts index a624d4db9b..9369f57413 100644 --- a/resources/benchmark/workers.ts +++ b/resources/benchmark/workers.ts @@ -3,8 +3,8 @@ import childProcess from 'node:child_process'; import { localRepoPath } from '../utils.js'; -import { nodeFlags } from './config.js'; -import type { BenchmarkSample } from './types.js'; +import { memoryBenchmarkNodeFlags } from './config.js'; +import type { BenchmarkTimingSample } from './types.js'; export function getBenchmarkName(modulePath: string): string { return runWorkerFile( @@ -13,17 +13,24 @@ export function getBenchmarkName(modulePath: string): string { ) as string; } -export function sampleModule(modulePath: string): BenchmarkSample { +export function sampleTimingModule(modulePath: string): BenchmarkTimingSample { return runWorkerFile( localRepoPath('resources/benchmark/worker-timing.js'), modulePath, - ) as BenchmarkSample; + ) as BenchmarkTimingSample; +} + +export function sampleMemoryModule(modulePath: string): number { + return runWorkerFile( + localRepoPath('resources/benchmark/worker-memory.js'), + modulePath, + ) as number; } function runWorkerFile(workerPath: string, modulePath: string): unknown { const result = childProcess.spawnSync( process.execPath, - [...nodeFlags, workerPath, modulePath], + [...memoryBenchmarkNodeFlags, workerPath, modulePath], { stdio: ['inherit', 'inherit', 'inherit', 'pipe'], env: { NODE_ENV: 'production' },