diff --git a/resources/benchmark/config.ts b/resources/benchmark/config.ts index d4fab34277..b241a2751b 100644 --- a/resources/benchmark/config.ts +++ b/resources/benchmark/config.ts @@ -5,3 +5,12 @@ export const LOCAL = 'local'; export const maxTime = 5; // The minimum sample size required to perform statistical analysis. export const minSamples = 5; + +export const nodeFlags: ReadonlyArray = [ + '--predictable', + '--no-concurrent-sweeping', + '--no-minor-gc-task', + '--min-semi-space-size=1280', // 1.25GB + '--max-semi-space-size=1280', // 1.25GB + '--trace-gc', // no gc calls should happen during benchmark, so trace them +]; diff --git a/resources/benchmark/sampling.ts b/resources/benchmark/sampling.ts index c06e4efaf2..bcb3879697 100644 --- a/resources/benchmark/sampling.ts +++ b/resources/benchmark/sampling.ts @@ -1,10 +1,11 @@ import assert from 'node:assert'; -import cp from 'node:child_process'; -import url from 'node:url'; import { maxTime, minSamples } from './config.js'; import { yellow } from './output.js'; import type { BenchmarkSample } from './types.js'; +import { sampleModule } from './workers.js'; + +export { sampleModule }; export function collectSamples(modulePath: string): Array { let numOfConsequentlyRejectedSamples = 0; @@ -35,72 +36,3 @@ export function collectSamples(modulePath: string): Array { } return samples; } - -export function sampleModule(modulePath: string): BenchmarkSample { - // To support Windows we need to use URL instead of path - const moduleURL = url.pathToFileURL(modulePath); - - const sampleCode = ` - import fs from 'node:fs'; - - import { benchmark } from '${moduleURL}'; - - // warm up, it looks like 7 is a magic number to reliably trigger JIT - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - await benchmark.measure(); - - const memBaseline = process.memoryUsage().heapUsed; - - const resourcesStart = process.resourceUsage(); - const startTime = process.hrtime.bigint(); - for (let i = 0; i < benchmark.count; ++i) { - await benchmark.measure(); - } - const timeDiff = Number(process.hrtime.bigint() - startTime); - const resourcesEnd = process.resourceUsage(); - - const result = { - name: benchmark.name, - clocked: timeDiff / benchmark.count, - memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count, - involuntaryContextSwitches: - resourcesEnd.involuntaryContextSwitches - resourcesStart.involuntaryContextSwitches, - }; - fs.writeFileSync(3, JSON.stringify(result)); - `; - - const result = cp.spawnSync( - process.execPath, - [ - // V8 flags - '--predictable', - '--no-concurrent-sweeping', - '--no-minor-gc-task', - '--min-semi-space-size=1280', // 1.25GB - '--max-semi-space-size=1280', // 1.25GB - '--trace-gc', // no gc calls should happen during benchmark, so trace them - - // Node.js flags - '--input-type=module', - '--eval', - sampleCode, - ], - { - stdio: ['inherit', 'inherit', 'inherit', 'pipe'], - env: { NODE_ENV: 'production' }, - }, - ); - - if (result.status !== 0) { - throw new Error(`Benchmark failed with "${result.status}" status.`); - } - - const resultStr = result.output[3]?.toString(); - assert(resultStr != null); - return JSON.parse(resultStr); -} diff --git a/resources/benchmark/worker-timing.js b/resources/benchmark/worker-timing.js new file mode 100644 index 0000000000..a02ee59a2a --- /dev/null +++ b/resources/benchmark/worker-timing.js @@ -0,0 +1,42 @@ +import { + loadBenchmark, + readModulePath, + runWorker, + writeResult, +} from './worker-utils.js'; + +runWorker(async () => { + const benchmark = await loadBenchmark(readModulePath()); + await warmUp(benchmark); + + const memBaseline = process.memoryUsage().heapUsed; + + const resourcesStart = process.resourceUsage(); + const startTime = process.hrtime.bigint(); + for (let i = 0; i < benchmark.count; ++i) { + // eslint-disable-next-line no-await-in-loop + await benchmark.measure(); + } + const timeDiff = Number(process.hrtime.bigint() - startTime); + const resourcesEnd = process.resourceUsage(); + + writeResult({ + name: benchmark.name, + clocked: timeDiff / benchmark.count, + memUsed: (process.memoryUsage().heapUsed - memBaseline) / benchmark.count, + involuntaryContextSwitches: + resourcesEnd.involuntaryContextSwitches - + resourcesStart.involuntaryContextSwitches, + }); +}); + +async function warmUp(benchmark) { + // It looks like 7 is a magic number to reliably trigger JIT. + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); + await benchmark.measure(); +} diff --git a/resources/benchmark/worker-utils.js b/resources/benchmark/worker-utils.js new file mode 100644 index 0000000000..c554c1fa77 --- /dev/null +++ b/resources/benchmark/worker-utils.js @@ -0,0 +1,33 @@ +import assert from 'node:assert'; +import fs from 'node:fs'; +import url from 'node:url'; + +export function readModulePath() { + const [modulePath] = process.argv.slice(2); + assert(modulePath != null); + return modulePath; +} + +export async function loadBenchmark(modulePath) { + const moduleURL = url.pathToFileURL(modulePath); + const module = await import(moduleURL.href); + const benchmark = module.benchmark; + if (benchmark?.name == null) { + throw new Error(`Benchmark at ${modulePath} must define a name.`); + } + assert(typeof benchmark.measure === 'function'); + return benchmark; +} + +export function writeResult(result) { + fs.writeFileSync(3, JSON.stringify(result)); +} + +export function runWorker(main) { + main().catch((error) => { + const errorMessage = + error instanceof Error ? (error.stack ?? error.message) : String(error); + process.stderr.write(errorMessage + '\n'); + process.exitCode = 1; + }); +} diff --git a/resources/benchmark/workers.ts b/resources/benchmark/workers.ts new file mode 100644 index 0000000000..cb6ac382f1 --- /dev/null +++ b/resources/benchmark/workers.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert'; +import childProcess from 'node:child_process'; + +import { localRepoPath } from '../utils.js'; + +import { nodeFlags } from './config.js'; +import type { BenchmarkSample } from './types.js'; + +export function sampleModule(modulePath: string): BenchmarkSample { + return runWorkerFile( + localRepoPath('resources/benchmark/worker-timing.js'), + modulePath, + ) as BenchmarkSample; +} + +function runWorkerFile(workerPath: string, modulePath: string): unknown { + const result = childProcess.spawnSync( + process.execPath, + [...nodeFlags, workerPath, modulePath], + { + stdio: ['inherit', 'inherit', 'inherit', 'pipe'], + env: { NODE_ENV: 'production' }, + }, + ); + if (result.status !== 0) { + throw new Error(`Benchmark worker failed with "${result.status}" status.`); + } + + const resultStr = result.output[3]?.toString(); + assert(resultStr != null); + return JSON.parse(resultStr); +}