From 89b94bde8166c18891e911f16dc757fa5b948942 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Wed, 20 May 2026 16:55:00 +0000 Subject: [PATCH 1/2] feat: add minimal BrowserStack wasm bench Add a minimal direct-wasm Chonk browser harness under barretenberg/wasm-bench. Generate one-off BrowserStack bench links, bot-friendly /5/worker JSON, and gist-preview HTML artifacts instead of running Selenium/BrowserStack Local from the package. Keep focused link/server tests and documentation for the claudebox BrowserStack workflow. --- .../skills/wasm-bench-browserstack/SKILL.md | 93 +++ barretenberg/wasm-bench/.yarnrc.yml | 7 + barretenberg/wasm-bench/README.md | 76 +++ barretenberg/wasm-bench/package.json | 22 + barretenberg/wasm-bench/scripts/build.mjs | 60 ++ .../wasm-bench/scripts/create-link.mjs | 85 +++ barretenberg/wasm-bench/scripts/lib.mjs | 276 ++++++++ .../wasm-bench/scripts/serve-bench.mjs | 229 +++++++ barretenberg/wasm-bench/scripts/test.mjs | 123 ++++ barretenberg/wasm-bench/src/bbapi.js | 26 + barretenberg/wasm-bench/src/bench-worker.js | 170 +++++ barretenberg/wasm-bench/src/chonk.js | 109 ++++ barretenberg/wasm-bench/src/crs.js | 29 + barretenberg/wasm-bench/src/index.html | 120 ++++ barretenberg/wasm-bench/src/main.js | 93 +++ barretenberg/wasm-bench/src/thread-worker.js | 37 ++ barretenberg/wasm-bench/src/wasm-runtime.js | 234 +++++++ .../wasm-bench/wasm-bench.config.json | 74 +++ barretenberg/wasm-bench/yarn.lock | 605 ++++++++++++++++++ 19 files changed, 2468 insertions(+) create mode 100644 .claude/skills/wasm-bench-browserstack/SKILL.md create mode 100644 barretenberg/wasm-bench/.yarnrc.yml create mode 100644 barretenberg/wasm-bench/README.md create mode 100644 barretenberg/wasm-bench/package.json create mode 100644 barretenberg/wasm-bench/scripts/build.mjs create mode 100644 barretenberg/wasm-bench/scripts/create-link.mjs create mode 100644 barretenberg/wasm-bench/scripts/lib.mjs create mode 100644 barretenberg/wasm-bench/scripts/serve-bench.mjs create mode 100644 barretenberg/wasm-bench/scripts/test.mjs create mode 100644 barretenberg/wasm-bench/src/bbapi.js create mode 100644 barretenberg/wasm-bench/src/bench-worker.js create mode 100644 barretenberg/wasm-bench/src/chonk.js create mode 100644 barretenberg/wasm-bench/src/crs.js create mode 100644 barretenberg/wasm-bench/src/index.html create mode 100644 barretenberg/wasm-bench/src/main.js create mode 100644 barretenberg/wasm-bench/src/thread-worker.js create mode 100644 barretenberg/wasm-bench/src/wasm-runtime.js create mode 100644 barretenberg/wasm-bench/wasm-bench.config.json create mode 100644 barretenberg/wasm-bench/yarn.lock diff --git a/.claude/skills/wasm-bench-browserstack/SKILL.md b/.claude/skills/wasm-bench-browserstack/SKILL.md new file mode 100644 index 000000000000..31c93b0a7015 --- /dev/null +++ b/.claude/skills/wasm-bench-browserstack/SKILL.md @@ -0,0 +1,93 @@ +--- +name: wasm-bench-browserstack +description: Create and publish one-off Chonk direct-wasm benchmark links for BrowserStack real browsers using pinned IVC inputs. +--- + +# BrowserStack Direct-Wasm Chonk Benchmark Links + +Use this skill for BrowserStack, mobile-browser, iOS/Android browser, or wasm +Chonk benchmark requests. The source of truth is `barretenberg/wasm-bench/`. +Do not extend `barretenberg/acir_tests/browser-test-app` for pure benchmark +work; that app is only for SDK/browser correctness coverage. + +## Harness + +`barretenberg/wasm-bench` is intentionally minimal: + +- no `@aztec/bb.js`, no Comlink, no browser-test-app +- bundles three browser modules: UI, benchmark worker, pthread worker +- loads `barretenberg.wasm.gz` or `barretenberg-threads.wasm.gz` directly +- calls the `bbapi` msgpack export for `SrsInitSrs`, `SrsInitGrumpkinSrs`, + `ChonkStart`, `ChonkLoad`, `ChonkAccumulate`, `ChonkProve`, + `ChonkComputeVk`, and `ChonkVerify` +- serves pinned `ivc-inputs.msgpack` from + `yarn-project/end-to-end/example-app-ivc-inputs-out` + +## Build + +```bash +cd barretenberg/cpp +AVM=0 AVM_TRANSPILER=0 cmake --preset wasm -DAVM=OFF -DAVM_TRANSPILER_LIB= -DENABLE_WASM_BENCH=ON +AVM=0 AVM_TRANSPILER=0 cmake --build --preset wasm --target barretenberg.wasm.gz -j 8 +AVM=0 AVM_TRANSPILER=0 cmake --preset wasm-threads -DAVM=OFF -DAVM_TRANSPILER_LIB= +AVM=0 AVM_TRANSPILER=0 cmake --build --preset wasm-threads --target barretenberg.wasm.gz -j 8 + +cd ../wasm-bench +yarn install +yarn build +``` + +`yarn build` fails clearly if either wasm artifact is missing. The threaded +preset already enables `ENABLE_WASM_BENCH`; pass it explicitly for the +single-thread preset. + +## Link Workflow + +Serve the built harness through a public tunnel, then create one-off links: + +```bash +cd barretenberg/wasm-bench +yarn create-link \ + --url https://.trycloudflare.com \ + --matrix customer-balanced \ + --html /tmp/wasm-bench-links.html \ + --json /tmp/wasm-bench-links.json \ + --format json +``` + +The script only creates artifacts. It does not install Selenium, start +BrowserStack Local, or hold BrowserStack credentials in the shell. The JSON is +bot-friendly: + +- `targets[].benchUrl` is the one-off `index.html?bench=` link. +- `targets[].browserstackWorker` is the BrowserStack `/5/worker` request body. +- `html.path`, when supplied, is the HTML page to publish with `cloxy-gist`. + +Publish the HTML with `cloxy-gist`, then post the raw gist file through +`https://htmlpreview.github.io/?`. To have the script print +that preview URL alongside the bench links, rerun it with `--gist-raw-url`. + +List target presets: + +```bash +yarn create-link --target true +``` + +Important target behavior: + +- `iphone-15-pro` injects `memMaxPages: 16384` because iOS Safari rejects the + default 4 GiB shared memory maximum. +- `customer-balanced` runs `macos`, `iphone-15-pro`, `galaxy-s25-ultra`, and + `pixel-9-pro-xl` in series. +- BrowserStack account concurrency is normally one; bots should create one + worker at a time unless account status says it is safe to fan out. + +## Reporting + +Always report `proveTotalMs = chonk_setup + chonk_prove` as the headline +metric. `proveMs` alone excludes the load/accumulate setup that is real Chonk +prover cost, while total wall time includes harness fetch, CRS transfer, SRS +load, and BrowserStack noise. + +Keep raw result JSONL, screenshots, the generated link JSON, and the HTML +preview gist with the PR or Slack report when sharing results. diff --git a/barretenberg/wasm-bench/.yarnrc.yml b/barretenberg/wasm-bench/.yarnrc.yml new file mode 100644 index 000000000000..1f1d24dfab5a --- /dev/null +++ b/barretenberg/wasm-bench/.yarnrc.yml @@ -0,0 +1,7 @@ +nodeLinker: node-modules + +installStatePath: /dev/null + +npmMinimalAgeGate: 7d + +npmPreapprovedPackages: [] diff --git a/barretenberg/wasm-bench/README.md b/barretenberg/wasm-bench/README.md new file mode 100644 index 000000000000..249fd9ee0f8d --- /dev/null +++ b/barretenberg/wasm-bench/README.md @@ -0,0 +1,76 @@ +# Barretenberg Wasm Bench + +Minimal direct-wasm browser harness for Chonk proving. It loads +`barretenberg.wasm.gz` directly, talks to the `bbapi` msgpack export, and runs +pinned `ivc-inputs.msgpack` flows without going through `@aztec/bb.js` or the +`acir_tests/browser-test-app` bundle. + +## Build + +```bash +cd barretenberg/cpp +AVM=0 AVM_TRANSPILER=0 cmake --preset wasm -DAVM=OFF -DAVM_TRANSPILER_LIB= -DENABLE_WASM_BENCH=ON +AVM=0 AVM_TRANSPILER=0 cmake --build --preset wasm --target barretenberg.wasm.gz -j 8 +AVM=0 AVM_TRANSPILER=0 cmake --preset wasm-threads -DAVM=OFF -DAVM_TRANSPILER_LIB= +AVM=0 AVM_TRANSPILER=0 cmake --build --preset wasm-threads --target barretenberg.wasm.gz -j 8 + +cd ../wasm-bench +yarn install +yarn build +``` + +## Serve Locally + +```bash +cd barretenberg/wasm-bench +yarn serve -- --port 8090 +``` + +Open `http://127.0.0.1:8090/index.html?bench=` or use the +default UI button. The server writes progress and result JSONL when +`--progress-jsonl` and `--result-jsonl` are supplied. + +## One-Off Links + +The package does not run BrowserStack directly. It creates one-off bench links +and BrowserStack `/5/worker` JSON that bots can publish or pass to the +BrowserStack MCP tools. + +```bash +cd barretenberg/wasm-bench +yarn create-link \ + --url https://.trycloudflare.com \ + --matrix customer-balanced \ + --html /tmp/wasm-bench-links.html \ + --json /tmp/wasm-bench-links.json \ + --format json +``` + +Publish the HTML and JSON with `cloxy-gist`, then post the raw HTML file through +an HTML preview link: + +```bash +cloxy-gist --description "wasm bench links" \ + bench-links.html=/tmp/wasm-bench-links.html \ + bench-links.json=/tmp/wasm-bench-links.json + +yarn create-link \ + --url https://.trycloudflare.com \ + --matrix customer-balanced \ + --gist-raw-url https://gist.githubusercontent.com//raw/bench-links.html +``` + +The generated JSON includes `targets[].benchUrl` and +`targets[].browserstackWorker`. Pass the worker JSON directly to +`browserstack_create_worker` when driving a target from claudebox. The headline +metric for completed prove results is: + +```text +proveTotalMs = chonk_setup + chonk_prove +``` + +List supported targets: + +```bash +yarn create-link --target true +``` diff --git a/barretenberg/wasm-bench/package.json b/barretenberg/wasm-bench/package.json new file mode 100644 index 000000000000..47c33307b097 --- /dev/null +++ b/barretenberg/wasm-bench/package.json @@ -0,0 +1,22 @@ +{ + "name": "@aztec/wasm-bench", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "node scripts/build.mjs", + "create-link": "node scripts/create-link.mjs", + "serve": "node scripts/serve-bench.mjs", + "test": "node scripts/test.mjs" + }, + "dependencies": { + "commander": "^12.1.0", + "esbuild": "^0.25.12", + "msgpackr": "^1.11.2", + "pako": "^2.1.0" + }, + "devDependencies": { + "@types/node": "20" + }, + "packageManager": "yarn@4.13.0" +} diff --git a/barretenberg/wasm-bench/scripts/build.mjs b/barretenberg/wasm-bench/scripts/build.mjs new file mode 100644 index 000000000000..6105c60eabd9 --- /dev/null +++ b/barretenberg/wasm-bench/scripts/build.mjs @@ -0,0 +1,60 @@ +import { copyFile, mkdir, rm, stat } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { build } from 'esbuild'; + +import { defaultDistDir, packageRoot, repoRoot } from './lib.mjs'; + +const distDir = resolve(process.env.WASM_BENCH_DIST_DIR || defaultDistDir); +const wasmOutputs = [ + { + source: resolve(repoRoot, 'barretenberg/cpp/build-wasm/bin/barretenberg.wasm.gz'), + dest: 'barretenberg.wasm.gz', + label: 'single-thread wasm', + command: 'cd barretenberg/cpp && cmake --preset wasm -DENABLE_WASM_BENCH=ON && cmake --build --preset wasm --target barretenberg.wasm.gz', + }, + { + source: resolve(repoRoot, 'barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm.gz'), + dest: 'barretenberg-threads.wasm.gz', + label: 'threaded wasm', + command: 'cd barretenberg/cpp && cmake --preset wasm-threads && cmake --build --preset wasm-threads --target barretenberg.wasm.gz', + }, +]; + +async function assertReadable(path, label, command) { + try { + const info = await stat(path); + if (!info.isFile() || info.size === 0) { + throw new Error(`${label} is empty`); + } + } catch (error) { + throw new Error(`${label} not found at ${path}.\nBuild it first:\n ${command}`, { cause: error }); + } +} + +await rm(distDir, { recursive: true, force: true }); +await mkdir(resolve(distDir, 'wasm'), { recursive: true }); + +await build({ + entryPoints: [ + resolve(packageRoot, 'src/main.js'), + resolve(packageRoot, 'src/bench-worker.js'), + resolve(packageRoot, 'src/thread-worker.js'), + ], + outdir: distDir, + entryNames: '[name]', + bundle: true, + format: 'esm', + target: ['es2022'], + sourcemap: true, + logLevel: 'info', +}); + +await copyFile(resolve(packageRoot, 'src/index.html'), resolve(distDir, 'index.html')); + +for (const output of wasmOutputs) { + await assertReadable(output.source, output.label, output.command); + await copyFile(output.source, resolve(distDir, 'wasm', output.dest)); +} + +console.log(`Built wasm bench into ${distDir}`); diff --git a/barretenberg/wasm-bench/scripts/create-link.mjs b/barretenberg/wasm-bench/scripts/create-link.mjs new file mode 100644 index 000000000000..cfb19b59c441 --- /dev/null +++ b/barretenberg/wasm-bench/scripts/create-link.mjs @@ -0,0 +1,85 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { Command } from 'commander'; + +import { + createLinkPlan, + defaultHtmlPreviewBase, + formatLinkPlanText, + listTargetLines, + loadConfig, + parseBoolean, + parsePositiveInteger, + renderPreviewHtml, +} from './lib.mjs'; + +function parseThreads(value) { + if (value === 'auto') { + return value; + } + return parsePositiveInteger(value, 'threads'); +} + +async function writeTextFile(path, body) { + if (!path) { + return; + } + const resolved = resolve(path); + await mkdir(dirname(resolved), { recursive: true }); + await writeFile(resolved, body); +} + +export async function main(argv = process.argv) { + const config = loadConfig(); + const program = new Command(); + program + .option('--url ', 'Served wasm-bench origin or index.html URL') + .option('--target ', 'Target name, comma list, matrix name, or "true" to list targets', 'macos') + .option('--matrix ', 'Matrix name or comma-separated target list') + .option('--flow ', 'Pinned Chonk input flow', config.defaultFlow) + .option('--runs ', 'Runs per target', value => parsePositiveInteger(value, 'runs'), config.defaultRuns) + .option('--threads ', 'Thread count or "auto"', parseThreads, config.defaultThreads) + .option('--smoke ', 'Stop after wasm/input initialization', value => parseBoolean(value), parseBoolean(process.env.WASM_BENCH_SMOKE, false)) + .option('--mem-max-pages ', 'Override WebAssembly.Memory maximum pages', value => parsePositiveInteger(value, 'mem-max-pages')) + .option('--srs-size ', 'Override BN254 SRS point count', value => parsePositiveInteger(value, 'srs-size')) + .option('--grumpkin-srs-size ', 'Override Grumpkin SRS point count', value => parsePositiveInteger(value, 'grumpkin-srs-size')) + .option('--timeout ', 'BrowserStack /5/worker timeout', value => parsePositiveInteger(value, 'timeout'), 1800) + .option('--name ', 'BrowserStack worker name') + .option('--build ', 'BrowserStack worker build name') + .option('--html ', 'Write gist-preview HTML link page') + .option('--json ', 'Write bot-friendly JSON link plan') + .option('--gist-raw-url ', 'Raw gist URL for the generated HTML file') + .option('--preview-base ', 'HTML preview service prefix', defaultHtmlPreviewBase) + .option('--format ', 'Stdout format: text or json', 'text') + .parse(argv); + + const options = program.opts(); + if (options.target === 'true') { + console.log(listTargetLines(config).join('\n')); + return undefined; + } + + const htmlPath = options.html ? resolve(options.html) : undefined; + const plan = createLinkPlan(config, { ...options, htmlPath, previewBase: options.previewBase }); + const json = `${JSON.stringify(plan, null, 2)}\n`; + await writeTextFile(options.json, json); + await writeTextFile(htmlPath, renderPreviewHtml(plan)); + + if (options.format === 'json') { + process.stdout.write(json); + } else if (options.format === 'text') { + process.stdout.write(formatLinkPlanText(plan)); + } else { + throw new Error(`Unknown --format ${options.format}; expected text or json`); + } + return plan; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/barretenberg/wasm-bench/scripts/lib.mjs b/barretenberg/wasm-bench/scripts/lib.mjs new file mode 100644 index 000000000000..f44130750309 --- /dev/null +++ b/barretenberg/wasm-bench/scripts/lib.mjs @@ -0,0 +1,276 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const scriptDir = dirname(fileURLToPath(import.meta.url)); +export const packageRoot = resolve(scriptDir, '..'); +export const repoRoot = resolve(packageRoot, '../..'); +export const configPath = resolve(packageRoot, 'wasm-bench.config.json'); +export const defaultDistDir = resolve(packageRoot, 'dest'); +export const defaultInputsDir = resolve(repoRoot, 'yarn-project/end-to-end/example-app-ivc-inputs-out'); +export const defaultHtmlPreviewBase = 'https://htmlpreview.github.io/?'; + +export function loadConfig(path = configPath) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +export function parseBoolean(value, defaultValue = false) { + if (value === undefined || value === null || value === '') { + return defaultValue; + } + if (typeof value === 'boolean') { + return value; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`Invalid boolean value: ${value}`); +} + +export function parsePositiveInteger(value, name) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, got ${value}`); + } + return parsed; +} + +export function encodeBenchParam(value) { + return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url'); +} + +export function decodeBenchParam(value) { + return JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); +} + +export function safeResolve(root, requestPath) { + if (requestPath.includes('\0')) { + throw new Error('Path contains a NUL byte'); + } + const normalizedRoot = resolve(root); + const relativeRequest = requestPath.startsWith('/') ? `.${requestPath}` : requestPath; + const resolved = resolve(normalizedRoot, decodeURIComponent(relativeRequest)); + const diff = relative(normalizedRoot, resolved); + if (diff === '' || (diff !== '..' && !diff.startsWith(`..${sep}`) && !isAbsolute(diff))) { + return resolved; + } + throw new Error(`Path escapes root: ${requestPath}`); +} + +export function getTarget(config, targetName) { + const target = config.targets[targetName]; + if (!target) { + const available = Object.keys(config.targets).sort().join(', '); + throw new Error(`Unknown target "${targetName}". Available targets: ${available}`); + } + return target; +} + +export function listTargetLines(config) { + return Object.entries(config.targets) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, target]) => `${name}\t${target.label}\t${target.chip}`); +} + +export function resolveTargetNames(config, { target, matrix }) { + if (matrix) { + const configured = config.matrices[matrix]; + if (configured) { + return configured; + } + return matrix.split(',').map(value => value.trim()).filter(Boolean); + } + const raw = target || 'macos'; + if (config.matrices[raw]) { + return config.matrices[raw]; + } + return raw.split(',').map(value => value.trim()).filter(Boolean); +} + +export function withBenchParam(baseUrl, benchOptions) { + const url = new URL(baseUrl); + if (url.pathname === '' || url.pathname === '/') { + url.pathname = '/index.html'; + } + url.searchParams.set('bench', encodeBenchParam(benchOptions)); + return url.toString(); +} + +export function htmlPreviewUrlForRawUrl(rawUrl, previewBase = defaultHtmlPreviewBase) { + if (!rawUrl) { + return undefined; + } + if (previewBase.endsWith('?')) { + return `${previewBase}${rawUrl}`; + } + if (previewBase.endsWith('=')) { + return `${previewBase}${encodeURIComponent(rawUrl)}`; + } + const joiner = previewBase.includes('?') ? '&url=' : '?url='; + return `${previewBase}${joiner}${encodeURIComponent(rawUrl)}`; +} + +export function createBenchOptions(config, targetName, options = {}) { + const target = getTarget(config, targetName); + return { + benchmark: config.defaultBenchmark, + flow: options.flow ?? config.defaultFlow, + runs: options.runs ?? config.defaultRuns, + threads: options.threads ?? config.defaultThreads, + smoke: Boolean(options.smoke), + ...(options.srsSize ? { srsSize: options.srsSize } : {}), + ...(options.grumpkinSrsSize ? { grumpkinSrsSize: options.grumpkinSrsSize } : {}), + ...(target.benchOverrides ?? {}), + ...(options.memMaxPages ? { memMaxPages: options.memMaxPages } : {}), + }; +} + +export function createBrowserStackWorkerBody(targetName, target, benchUrl, options = {}) { + if (!target.browserstackWorker) { + return undefined; + } + return { + ...structuredClone(target.browserstackWorker), + url: benchUrl, + timeout: options.timeout ?? 1800, + name: options.name ?? `wasm-bench ${targetName}`, + build: options.build ?? `wasm-bench-${new Date().toISOString().slice(0, 10)}`, + }; +} + +export function createLinkPlan(config, options = {}) { + const targetNames = resolveTargetNames(config, options); + const generatedAt = options.generatedAt ?? new Date().toISOString(); + const baseUrl = options.url; + if (!baseUrl) { + throw new Error('Pass --url with the served wasm-bench origin.'); + } + const htmlPreviewUrl = htmlPreviewUrlForRawUrl(options.gistRawUrl, options.previewBase); + return { + generatedAt, + baseUrl, + html: { + ...(options.htmlPath ? { path: options.htmlPath } : {}), + ...(options.gistRawUrl ? { rawUrl: options.gistRawUrl } : {}), + ...(htmlPreviewUrl ? { previewUrl: htmlPreviewUrl } : {}), + }, + targets: targetNames.map(targetName => { + const target = getTarget(config, targetName); + const benchOptions = createBenchOptions(config, targetName, options); + const benchUrl = withBenchParam(baseUrl, benchOptions); + return { + target: targetName, + label: target.label, + chip: target.chip, + firstProgressMs: target.firstProgressMs, + benchOptions, + benchUrl, + browserstackWorker: createBrowserStackWorkerBody(targetName, target, benchUrl, options), + }; + }), + }; +} + +function escapeHtml(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function jsonForScript(value) { + return JSON.stringify(value, null, 2).replaceAll('<', '\\u003c'); +} + +export function renderPreviewHtml(plan) { + const rows = plan.targets + .map(row => ` +
+

${escapeHtml(row.target)}

+

${escapeHtml(row.label)} - ${escapeHtml(row.chip)}

+ Open bench link +
+ BrowserStack worker JSON + 
${escapeHtml(JSON.stringify(row.browserstackWorker, null, 2))}
+
+
`) + .join('\n'); + + return ` + + + + + Barretenberg Wasm Bench Links + + + +
+

Barretenberg Wasm Bench Links

+

Generated ${escapeHtml(plan.generatedAt)} from ${escapeHtml(plan.baseUrl)}

+${rows} +
+ + + +`; +} + +export function formatLinkPlanText(plan) { + const lines = []; + if (plan.html.previewUrl) { + lines.push(`preview\t${plan.html.previewUrl}`); + } + for (const row of plan.targets) { + lines.push(`${row.target}\t${row.benchUrl}`); + } + return `${lines.join('\n')}\n`; +} + +export function proveTotalMs(run) { + if (Number.isFinite(run?.proveTotalMs)) { + return run.proveTotalMs; + } + return Number(run?.setupMs ?? 0) + Number(run?.proveMs ?? 0); +} + +export function formatHeadline(targetName, result) { + if (result?.smoke) { + return `${targetName}\tsmoke\twallMs=${Math.round(result.wallMs)}`; + } + const rows = result?.runs ?? []; + if (rows.length === 0) { + return `${targetName}\tno-runs`; + } + return rows + .map(run => `${targetName}\trun=${run.run}\tproveTotalMs=${Math.round(proveTotalMs(run))}\tsetupMs=${Math.round(run.setupMs)}\tproveMs=${Math.round(run.proveMs)}`) + .join('\n'); +} + +export function requireExistingFile(path, message) { + if (!existsSync(path)) { + throw new Error(message); + } + return path; +} + +export function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} diff --git a/barretenberg/wasm-bench/scripts/serve-bench.mjs b/barretenberg/wasm-bench/scripts/serve-bench.mjs new file mode 100644 index 000000000000..7c65559cb78a --- /dev/null +++ b/barretenberg/wasm-bench/scripts/serve-bench.mjs @@ -0,0 +1,229 @@ +import { createServer } from 'node:http'; +import { appendFile, mkdir, readFile, readdir, stat } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { basename, dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { Command } from 'commander'; + +import { + defaultDistDir, + defaultInputsDir, + parsePositiveInteger, + safeResolve, +} from './lib.mjs'; + +const CRS_PRIMARY_HOST = 'https://crs.aztec-cdn.foundation'; +const CRS_FALLBACK_HOST = 'https://crs.aztec-labs.com'; +const MAX_BODY_BYTES = 10 * 1024 * 1024; + +const contentTypes = new Map([ + ['.html', 'text/html; charset=utf-8'], + ['.js', 'text/javascript; charset=utf-8'], + ['.map', 'application/json; charset=utf-8'], + ['.json', 'application/json; charset=utf-8'], + ['.msgpack', 'application/octet-stream'], + ['.wasm', 'application/wasm'], + ['.gz', 'application/gzip'], +]); + +function setSharedHeaders(response) { + response.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + response.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); + response.setHeader('Cross-Origin-Resource-Policy', 'same-origin'); + response.setHeader('Cache-Control', 'no-store'); +} + +function send(response, status, body, headers = {}) { + setSharedHeaders(response); + response.writeHead(status, headers); + response.end(body); +} + +function sendJson(response, status, body) { + send(response, status, JSON.stringify(body), { 'Content-Type': 'application/json; charset=utf-8' }); +} + +function sendError(response, status, error) { + sendJson(response, status, { ok: false, error: error?.message ?? String(error) }); +} + +async function appendJsonl(path, body) { + if (!path) { + return; + } + await mkdir(dirname(path), { recursive: true }); + await appendFile(path, `${JSON.stringify({ receivedAt: new Date().toISOString(), ...body })}\n`); +} + +async function readJsonBody(request) { + const chunks = []; + let total = 0; + for await (const chunk of request) { + total += chunk.byteLength; + if (total > MAX_BODY_BYTES) { + throw new Error(`Request body exceeds ${MAX_BODY_BYTES} bytes`); + } + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); +} + +async function fetchWithFallback(primaryUrl, fallbackUrl, options) { + try { + const response = await fetch(primaryUrl, options); + if (response.ok || response.status === 206) { + return response; + } + throw new Error(`HTTP ${response.status}`); + } catch { + const fallback = await fetch(fallbackUrl, options); + if (fallback.ok || fallback.status === 206) { + return fallback; + } + throw new Error(`Failed CRS fetch: ${primaryUrl} and ${fallbackUrl} returned HTTP ${fallback.status}`); + } +} + +function rangeOptions(points, bytesPerPoint) { + const count = parsePositiveInteger(points, 'points'); + return { headers: { Range: `bytes=0-${count * bytesPerPoint - 1}` } }; +} + +async function proxyCrs(response, pathname, searchParams) { + let fileName; + let options = {}; + if (pathname === '/crs/bn254-g1') { + fileName = 'g1_compressed.dat'; + options = rangeOptions(searchParams.get('points') ?? '524288', 32); + } else if (pathname === '/crs/bn254-g2') { + fileName = 'g2.dat'; + } else if (pathname === '/crs/grumpkin-g1') { + fileName = 'grumpkin_g1.dat'; + options = rangeOptions(searchParams.get('points') ?? '65536', 64); + } else { + return false; + } + + const upstream = await fetchWithFallback(`${CRS_PRIMARY_HOST}/${fileName}`, `${CRS_FALLBACK_HOST}/${fileName}`, { + ...options, + cache: 'force-cache', + }); + const bytes = Buffer.from(await upstream.arrayBuffer()); + send(response, 200, bytes, { + 'Content-Type': 'application/octet-stream', + 'Content-Length': String(bytes.byteLength), + }); + return true; +} + +async function listInputs(inputsDir) { + const entries = await readdir(inputsDir, { withFileTypes: true }).catch(() => []); + const flows = []; + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + const file = join(inputsDir, entry.name, 'ivc-inputs.msgpack'); + if (await stat(file).then(info => info.isFile()).catch(() => false)) { + flows.push(entry.name); + } + } + return flows.sort(); +} + +async function serveStatic(response, root, pathname) { + const relativePath = pathname === '/' ? '/index.html' : pathname; + const filePath = safeResolve(root, relativePath); + const info = await stat(filePath); + if (!info.isFile()) { + send(response, 404, 'not found', { 'Content-Type': 'text/plain; charset=utf-8' }); + return; + } + setSharedHeaders(response); + response.writeHead(200, { + 'Content-Type': contentTypes.get(extname(filePath)) ?? contentTypes.get(extname(basename(filePath, '.gz'))) ?? 'application/octet-stream', + 'Content-Length': String(info.size), + }); + createReadStream(filePath).pipe(response); +} + +export function createBenchServer({ + root = defaultDistDir, + inputsDir = defaultInputsDir, + progressJsonl = '/tmp/wasm-bench-progress.jsonl', + resultJsonl = '/tmp/wasm-bench-results.jsonl', +} = {}) { + const staticRoot = resolve(root); + const inputRoot = resolve(inputsDir); + + return createServer(async (request, response) => { + try { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + if (request.method === 'GET' && url.pathname === '/health') { + sendJson(response, 200, { ok: true, root: staticRoot, inputsDir: inputRoot }); + return; + } + if (request.method === 'GET' && url.pathname === '/inputs/index.json') { + sendJson(response, 200, { flows: await listInputs(inputRoot) }); + return; + } + if (request.method === 'GET' && url.pathname.startsWith('/inputs/') && url.pathname.endsWith('/ivc-inputs.msgpack')) { + const flow = decodeURIComponent(url.pathname.slice('/inputs/'.length, -'/ivc-inputs.msgpack'.length)); + const filePath = safeResolve(inputRoot, `/${flow}/ivc-inputs.msgpack`); + const body = await readFile(filePath); + send(response, 200, body, { 'Content-Type': 'application/octet-stream', 'Content-Length': String(body.byteLength) }); + return; + } + if (request.method === 'GET' && await proxyCrs(response, url.pathname, url.searchParams)) { + return; + } + if (request.method === 'POST' && url.pathname === '/progress') { + const body = await readJsonBody(request); + await appendJsonl(progressJsonl, body); + sendJson(response, 200, { ok: true }); + return; + } + if (request.method === 'POST' && url.pathname === '/result') { + const body = await readJsonBody(request); + await appendJsonl(resultJsonl, body); + sendJson(response, 200, { ok: true }); + return; + } + if (request.method !== 'GET') { + send(response, 405, 'method not allowed', { 'Content-Type': 'text/plain; charset=utf-8' }); + return; + } + await serveStatic(response, staticRoot, url.pathname); + } catch (error) { + sendError(response, error?.code === 'ENOENT' ? 404 : 500, error); + } + }); +} + +export async function main(argv = process.argv) { + const program = new Command(); + program + .option('--host ', 'Bind host', process.env.WASM_BENCH_HOST || '127.0.0.1') + .option('--port ', 'Bind port', value => parsePositiveInteger(value, 'port'), Number(process.env.WASM_BENCH_PORT || 8090)) + .option('--root ', 'Built harness directory', process.env.WASM_BENCH_DIST_DIR || defaultDistDir) + .option('--inputs-dir ', 'Pinned Chonk inputs directory', process.env.CHONK_PINNED_IVC_INPUTS_DIR || defaultInputsDir) + .option('--progress-jsonl ', 'Progress JSONL output path', process.env.WASM_BENCH_PROGRESS_JSONL || '/tmp/wasm-bench-progress.jsonl') + .option('--result-jsonl ', 'Result JSONL output path', process.env.WASM_BENCH_RESULT_JSONL || '/tmp/wasm-bench-results.jsonl') + .parse(argv); + + const options = program.opts(); + const server = createBenchServer(options); + await new Promise((resolveListen, rejectListen) => { + server.once('error', rejectListen); + server.listen(options.port, options.host, resolveListen); + }); + console.log(`wasm-bench serving http://${options.host}:${options.port}`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/barretenberg/wasm-bench/scripts/test.mjs b/barretenberg/wasm-bench/scripts/test.mjs new file mode 100644 index 000000000000..f58bd617c00d --- /dev/null +++ b/barretenberg/wasm-bench/scripts/test.mjs @@ -0,0 +1,123 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + createLinkPlan, + decodeBenchParam, + encodeBenchParam, + formatHeadline, + htmlPreviewUrlForRawUrl, + getTarget, + loadConfig, + renderPreviewHtml, + resolveTargetNames, + safeResolve, +} from './lib.mjs'; +import { createBenchServer } from './serve-bench.mjs'; + +test('base64url bench parameters round-trip JSON', () => { + const value = { + flow: 'ecdsar1+transfer_1_recursions+sponsored_fpc', + runs: 1, + threads: 'auto', + smoke: true, + }; + assert.deepEqual(decodeBenchParam(encodeBenchParam(value)), value); +}); + +test('config exposes target presets and matrices', () => { + const config = loadConfig(); + assert.equal(getTarget(config, 'iphone-15-pro').benchOverrides.memMaxPages, 16384); + assert.deepEqual(resolveTargetNames(config, { matrix: 'customer-balanced' }), [ + 'macos', + 'iphone-15-pro', + 'galaxy-s25-ultra', + 'pixel-9-pro-xl', + ]); +}); + +test('safeResolve rejects path traversal', async () => { + const root = await mkdtemp(join(tmpdir(), 'wasm-bench-root-')); + assert.equal(safeResolve(root, '/index.html'), join(root, 'index.html')); + assert.throws(() => safeResolve(root, '/../secret'), /escapes root/); +}); + +test('headline uses proveTotalMs as primary metric', () => { + assert.match( + formatHeadline('macos', { + runs: [{ run: 1, setupMs: 10.4, proveMs: 20.4 }], + }), + /proveTotalMs=31/, + ); +}); + +test('link plan creates one-off bench URLs and worker JSON', () => { + const config = loadConfig(); + const plan = createLinkPlan(config, { + url: 'https://example.com', + target: 'iphone-15-pro', + runs: 2, + threads: 4, + smoke: true, + generatedAt: '2026-05-20T00:00:00.000Z', + gistRawUrl: 'https://gist.githubusercontent.com/example/raw/bench-links.html', + }); + const [target] = plan.targets; + const url = new URL(target.benchUrl); + assert.equal(url.pathname, '/index.html'); + assert.equal(target.browserstackWorker.url, target.benchUrl); + assert.equal(target.browserstackWorker.device, 'iPhone 15 Pro'); + assert.equal(plan.html.previewUrl, 'https://htmlpreview.github.io/?https://gist.githubusercontent.com/example/raw/bench-links.html'); + assert.deepEqual(decodeBenchParam(url.searchParams.get('bench')), { + benchmark: 'chonk-prove', + flow: config.defaultFlow, + runs: 2, + threads: 4, + smoke: true, + memMaxPages: 16384, + }); +}); + +test('preview HTML embeds links and bot-readable JSON safely', () => { + const config = loadConfig(); + const plan = createLinkPlan(config, { + url: 'https://bench.invalid', + target: 'macos', + flow: '', + generatedAt: '2026-05-20T00:00:00.000Z', + }); + const html = renderPreviewHtml(plan); + assert.match(html, /Open bench link/); + assert.match(html, /wasm-bench-link-plan/); + assert.doesNotMatch(html, / + + diff --git a/barretenberg/wasm-bench/src/main.js b/barretenberg/wasm-bench/src/main.js new file mode 100644 index 000000000000..1a51d1abe7b2 --- /dev/null +++ b/barretenberg/wasm-bench/src/main.js @@ -0,0 +1,93 @@ +const logEl = document.querySelector('#log'); +const form = document.querySelector('#bench-form'); +const runButton = document.querySelector('#run'); + +function appendLog(value) { + const text = typeof value === 'string' ? value : JSON.stringify(value); + logEl.textContent += `${text}\n`; + logEl.scrollTop = logEl.scrollHeight; +} + +function decodeBenchParam(raw) { + const normalized = raw.replace(/-/g, '+').replace(/_/g, '/'); + const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, '='); + return JSON.parse(new TextDecoder().decode(Uint8Array.from(atob(padded), c => c.charCodeAt(0)))); +} + +async function postJson(path, body) { + try { + await fetch(path, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + } catch { + // Posting progress to the local harness is best effort. + } +} + +async function run(options) { + runButton.disabled = true; + logEl.textContent = ''; + window.__wasmBenchStatus = { state: 'running', options }; + window.__wasmBenchResult = undefined; + window.__wasmBenchError = undefined; + + const worker = new Worker(new URL('./bench-worker.js', import.meta.url), { type: 'module' }); + worker.onmessage = async event => { + const message = event.data; + if (message.type === 'progress') { + const row = { ...message, options }; + window.__wasmBenchStatus = { state: 'progress', event: message.event, data: message.data }; + appendLog(row); + await postJson('/progress', row); + return; + } + + if (message.type === 'result') { + window.__wasmBenchStatus = { state: 'complete' }; + window.__wasmBenchResult = message.result; + appendLog(message.result); + await postJson('/result', message.result); + worker.terminate(); + runButton.disabled = false; + return; + } + + if (message.type === 'error') { + window.__wasmBenchStatus = { state: 'error', error: message.error }; + window.__wasmBenchError = message.error; + appendLog(message.error); + await postJson('/progress', { type: 'error', error: message.error, options }); + worker.terminate(); + runButton.disabled = false; + } + }; + worker.onerror = async event => { + const error = { message: event.message, stack: event.error?.stack ?? '' }; + window.__wasmBenchStatus = { state: 'error', error }; + window.__wasmBenchError = error; + appendLog(error); + await postJson('/progress', { type: 'error', error, options }); + worker.terminate(); + runButton.disabled = false; + }; + worker.postMessage({ type: 'run', options }); +} + +form.addEventListener('submit', event => { + event.preventDefault(); + const formData = new FormData(form); + void run({ + flow: String(formData.get('flow')), + threads: String(formData.get('threads')), + runs: Number(formData.get('runs')), + smoke: String(formData.get('smoke')) === 'true', + }); +}); + +const params = new URLSearchParams(location.search); +const benchParam = params.get('bench'); +if (benchParam) { + void run(decodeBenchParam(benchParam)); +} diff --git a/barretenberg/wasm-bench/src/thread-worker.js b/barretenberg/wasm-bench/src/thread-worker.js new file mode 100644 index 000000000000..f15ffcdaef99 --- /dev/null +++ b/barretenberg/wasm-bench/src/thread-worker.js @@ -0,0 +1,37 @@ +import { createImportObject } from './wasm-runtime.js'; + +let instance; +let memory; + +self.addEventListener('message', async event => { + const message = event.data; + try { + if (message.type === 'init') { + memory = message.memory; + instance = await WebAssembly.instantiate( + message.module, + createImportObject({ + memory, + logger: text => self.postMessage({ type: 'log', message: text }), + envHardwareConcurrency: () => 1, + threadSpawn: () => { + throw new Error('WASM child threads cannot spawn nested threads.'); + }, + }), + ); + self.postMessage({ type: 'ready' }); + return; + } + + if (message.type === 'run') { + instance.exports.wasi_thread_start(message.id >>> 0, message.arg >>> 0); + return; + } + + if (message.type === 'destroy') { + self.close(); + } + } catch (error) { + self.postMessage({ type: 'error', message: error?.message ?? String(error), stack: error?.stack ?? '' }); + } +}); diff --git a/barretenberg/wasm-bench/src/wasm-runtime.js b/barretenberg/wasm-bench/src/wasm-runtime.js new file mode 100644 index 000000000000..c6fd1c179add --- /dev/null +++ b/barretenberg/wasm-bench/src/wasm-runtime.js @@ -0,0 +1,234 @@ +import pako from 'pako'; + +const WASM_PAGE_SIZE = 64 * 1024; +const MSGPACK_SCRATCH_SIZE = 8 * 1024 * 1024; + +function randomBytes(length) { + const out = new Uint8Array(length); + globalThis.crypto.getRandomValues(out); + return out; +} + +function getMemoryView(memory) { + return new Uint8Array(memory.buffer); +} + +function stringFromMemory(memory, addr) { + const mem = getMemoryView(memory); + let end = addr >>> 0; + while (mem[end] !== 0) { + end++; + } + return new TextDecoder('ascii').decode(mem.subarray(addr >>> 0, end)); +} + +export function createImportObject({ memory, logger = () => {}, envHardwareConcurrency = () => 1, threadSpawn }) { + return { + wasi_snapshot_preview1: { + random_get(out, length) { + getMemoryView(memory).set(randomBytes(length), out >>> 0); + }, + clock_time_get(_clockId, _precision, out) { + new DataView(memory.buffer).setBigUint64(out >>> 0, BigInt(Date.now()) * 1000000n, true); + }, + proc_exit(code) { + throw new Error(`WASI proc_exit(${code})`); + }, + }, + wasi: { + 'thread-spawn': threadSpawn ?? (() => { + throw new Error('WASM thread tried to spawn another thread'); + }), + }, + env: { + memory, + env_hardware_concurrency: envHardwareConcurrency, + logstr(addr) { + const text = stringFromMemory(memory, addr); + const mib = (memory.buffer.byteLength / (1024 * 1024)).toFixed(2); + logger(`${text} (mem: ${mib}MiB)`); + }, + throw_or_abort_impl(addr) { + throw new Error(stringFromMemory(memory, addr)); + }, + }, + }; +} + +async function fetchWasmBytes(url, progress) { + const started = performance.now(); + const response = await fetch(url, { cache: 'no-store' }); + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); + } + const compressed = new Uint8Array(await response.arrayBuffer()); + progress?.('wasm_fetch', { url, bytes: compressed.byteLength, elapsedMs: performance.now() - started }); + + const isGzip = compressed[0] === 0x1f && compressed[1] === 0x8b && compressed[2] === 0x08; + return isGzip ? pako.ungzip(compressed) : compressed; +} + +function copyOut(memory, start, end) { + return getMemoryView(memory).subarray(start >>> 0, end >>> 0).slice(); +} + +export class DirectBbWasm { + constructor({ logger = () => {}, progress = () => {} } = {}) { + this.logger = logger; + this.progress = progress; + this.instance = undefined; + this.memory = undefined; + this.threadWorkers = []; + this.nextThreadId = 1; + this.nextWorker = 0; + this.msgpackInputScratch = 0; + this.msgpackOutputScratch = 0; + } + + async init({ threads = 1, wasmBaseUrl = '/wasm', memInitialPages = 35, memMaxPages } = {}) { + const requestedThreads = Math.max(1, Number(threads) || 1); + const shared = requestedThreads > 1; + if (shared && (typeof SharedArrayBuffer === 'undefined' || !globalThis.crossOriginIsolated)) { + throw new Error('SharedArrayBuffer requires crossOriginIsolated=true for threaded wasm.'); + } + + const wasmName = shared ? 'barretenberg-threads.wasm.gz' : 'barretenberg.wasm.gz'; + const wasmUrl = `${wasmBaseUrl.replace(/\/$/, '')}/${wasmName}`; + const bytes = await fetchWasmBytes(wasmUrl, this.progress); + const compileStarted = performance.now(); + const module = await WebAssembly.compile(bytes); + this.progress('wasm_compile', { elapsedMs: performance.now() - compileStarted, bytes: bytes.byteLength }); + + const maximum = memMaxPages ?? (isIos() ? 2 ** 14 : 2 ** 16); + this.memory = new WebAssembly.Memory({ initial: memInitialPages, maximum, shared }); + + const imports = createImportObject({ + memory: this.memory, + logger: this.logger, + envHardwareConcurrency: () => this.threadWorkers.length + 1, + threadSpawn: arg => this.spawnThread(arg), + }); + + const instantiateStarted = performance.now(); + this.instance = await WebAssembly.instantiate(module, imports); + this.call('_initialize'); + this.msgpackInputScratch = this.call('bbmalloc', MSGPACK_SCRATCH_SIZE); + this.msgpackOutputScratch = this.call('bbmalloc', MSGPACK_SCRATCH_SIZE); + this.progress('wasm_instantiate', { + elapsedMs: performance.now() - instantiateStarted, + threads: requestedThreads, + shared, + memoryInitialMiB: (memInitialPages * WASM_PAGE_SIZE) / (1024 * 1024), + memoryMaxMiB: (maximum * WASM_PAGE_SIZE) / (1024 * 1024), + }); + + if (requestedThreads > 1) { + await this.initThreads(module, requestedThreads - 1); + } + } + + async initThreads(module, count) { + const workers = []; + for (let i = 0; i < count; i++) { + const worker = new Worker(new URL('./thread-worker.js', import.meta.url), { type: 'module' }); + workers.push(worker); + await new Promise((resolve, reject) => { + const onMessage = event => { + if (event.data?.type === 'ready') { + cleanup(); + resolve(); + } else if (event.data?.type === 'log') { + this.logger(event.data.message); + } else if (event.data?.type === 'error') { + cleanup(); + reject(new Error(event.data.message)); + } + }; + const onError = event => { + cleanup(); + reject(event.error ?? new Error(event.message)); + }; + const cleanup = () => { + worker.removeEventListener('message', onMessage); + worker.removeEventListener('error', onError); + }; + worker.addEventListener('message', onMessage); + worker.addEventListener('error', onError); + worker.postMessage({ type: 'init', module, memory: this.memory }); + }); + } + this.threadWorkers = workers; + this.progress('thread_workers_ready', { workers: workers.length }); + } + + spawnThread(arg) { + if (this.threadWorkers.length === 0) { + throw new Error('WASM requested a worker thread, but no thread workers are available.'); + } + const id = this.nextThreadId++; + const worker = this.threadWorkers[this.nextWorker++ % this.threadWorkers.length]; + worker.postMessage({ type: 'run', id, arg: arg >>> 0 }); + return id; + } + + exports() { + if (!this.instance) { + throw new Error('WASM is not initialized.'); + } + return this.instance.exports; + } + + call(name, ...args) { + const fn = this.exports()[name]; + if (typeof fn !== 'function') { + throw new Error(`WASM function ${name} not found.`); + } + return fn(...args) >>> 0; + } + + cbindCall(inputBuffer) { + if (!this.memory) { + throw new Error('WASM memory is not initialized.'); + } + + const needsCustomInputBuffer = inputBuffer.length > MSGPACK_SCRATCH_SIZE; + const inputPtr = needsCustomInputBuffer ? this.call('bbmalloc', inputBuffer.length) : this.msgpackInputScratch; + getMemoryView(this.memory).set(inputBuffer, inputPtr); + + const metadataSize = 8; + const outputPtrLocation = this.msgpackOutputScratch; + const outputSizeLocation = this.msgpackOutputScratch + 4; + const scratchDataPtr = this.msgpackOutputScratch + metadataSize; + const scratchDataSize = MSGPACK_SCRATCH_SIZE - metadataSize; + let view = new DataView(this.memory.buffer); + view.setUint32(outputPtrLocation, scratchDataPtr, true); + view.setUint32(outputSizeLocation, scratchDataSize, true); + + this.call('bbapi', inputPtr, inputBuffer.length, outputPtrLocation, outputSizeLocation); + + if (needsCustomInputBuffer) { + this.call('bbfree', inputPtr); + } + + view = new DataView(this.memory.buffer); + const outputDataPtr = view.getUint32(outputPtrLocation, true); + const outputSize = view.getUint32(outputSizeLocation, true); + const encodedResult = copyOut(this.memory, outputDataPtr, outputDataPtr + outputSize); + if (outputDataPtr !== scratchDataPtr) { + this.call('bbfree', outputDataPtr); + } + return encodedResult; + } + + async destroy() { + for (const worker of this.threadWorkers) { + worker.postMessage({ type: 'destroy' }); + worker.terminate(); + } + this.threadWorkers = []; + } +} + +function isIos() { + return typeof navigator !== 'undefined' && /iPad|iPhone/.test(navigator.userAgent); +} diff --git a/barretenberg/wasm-bench/wasm-bench.config.json b/barretenberg/wasm-bench/wasm-bench.config.json new file mode 100644 index 000000000000..34c51fc87517 --- /dev/null +++ b/barretenberg/wasm-bench/wasm-bench.config.json @@ -0,0 +1,74 @@ +{ + "defaultBenchmark": "chonk-prove", + "defaultFlow": "ecdsar1+transfer_1_recursions+sponsored_fpc", + "defaultRuns": 1, + "defaultThreads": "auto", + "defaultSrsSize": 524288, + "defaultGrumpkinSrsSize": 65536, + "targets": { + "macos": { + "label": "macOS Sequoia Chrome", + "chip": "Apple M-series BrowserStack desktop", + "firstProgressMs": 60000, + "browserstackWorker": { + "browser": "chrome", + "browser_version": "latest", + "os": "OS X", + "os_version": "Sequoia", + "resolution": "1920x1080" + } + }, + "windows-chrome": { + "label": "Windows Chrome", + "chip": "BrowserStack Windows desktop", + "firstProgressMs": 60000, + "browserstackWorker": { + "browser": "chrome", + "browser_version": "latest", + "os": "Windows", + "os_version": "11", + "resolution": "1920x1080" + } + }, + "iphone-15-pro": { + "label": "iPhone 15 Pro Safari", + "chip": "A17 Pro", + "firstProgressMs": 120000, + "benchOverrides": { + "memMaxPages": 16384 + }, + "browserstackWorker": { + "browser": "safari", + "device": "iPhone 15 Pro", + "os": "ios", + "os_version": "17" + } + }, + "galaxy-s25-ultra": { + "label": "Galaxy S25 Ultra Chrome", + "chip": "Snapdragon 8 Elite", + "firstProgressMs": 90000, + "browserstackWorker": { + "browser": "chrome", + "device": "Samsung Galaxy S25 Ultra", + "os": "android", + "os_version": "15.0" + } + }, + "pixel-9-pro-xl": { + "label": "Pixel 9 Pro XL Chrome", + "chip": "Google Tensor G4", + "firstProgressMs": 90000, + "browserstackWorker": { + "browser": "chrome", + "device": "Google Pixel 9 Pro XL", + "os": "android", + "os_version": "15.0" + } + } + }, + "matrices": { + "customer-balanced": ["macos", "iphone-15-pro", "galaxy-s25-ultra", "pixel-9-pro-xl"], + "all": ["macos", "windows-chrome", "iphone-15-pro", "galaxy-s25-ultra", "pixel-9-pro-xl"] + } +} diff --git a/barretenberg/wasm-bench/yarn.lock b/barretenberg/wasm-bench/yarn.lock new file mode 100644 index 000000000000..2a25fce6e925 --- /dev/null +++ b/barretenberg/wasm-bench/yarn.lock @@ -0,0 +1,605 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@aztec/wasm-bench@workspace:.": + version: 0.0.0-use.local + resolution: "@aztec/wasm-bench@workspace:." + dependencies: + "@types/node": "npm:20" + commander: "npm:^12.1.0" + esbuild: "npm:^0.25.12" + msgpackr: "npm:^1.11.2" + pako: "npm:^2.1.0" + languageName: unknown + linkType: soft + +"@esbuild/aix-ppc64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/aix-ppc64@npm:0.25.12" + conditions: os=aix & cpu=ppc64 + languageName: node + linkType: hard + +"@esbuild/android-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/android-arm64@npm:0.25.12" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/android-arm@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/android-arm@npm:0.25.12" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@esbuild/android-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/android-x64@npm:0.25.12" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/darwin-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/darwin-arm64@npm:0.25.12" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/darwin-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/darwin-x64@npm:0.25.12" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/freebsd-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/freebsd-arm64@npm:0.25.12" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/freebsd-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/freebsd-x64@npm:0.25.12" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/linux-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-arm64@npm:0.25.12" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/linux-arm@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-arm@npm:0.25.12" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@esbuild/linux-ia32@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-ia32@npm:0.25.12" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + +"@esbuild/linux-loong64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-loong64@npm:0.25.12" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + +"@esbuild/linux-mips64el@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-mips64el@npm:0.25.12" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + +"@esbuild/linux-ppc64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-ppc64@npm:0.25.12" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + +"@esbuild/linux-riscv64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-riscv64@npm:0.25.12" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + +"@esbuild/linux-s390x@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-s390x@npm:0.25.12" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + +"@esbuild/linux-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/linux-x64@npm:0.25.12" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/netbsd-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/netbsd-arm64@npm:0.25.12" + conditions: os=netbsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/netbsd-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/netbsd-x64@npm:0.25.12" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/openbsd-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/openbsd-arm64@npm:0.25.12" + conditions: os=openbsd & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/openbsd-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/openbsd-x64@npm:0.25.12" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/openharmony-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/openharmony-arm64@npm:0.25.12" + conditions: os=openharmony & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/sunos-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/sunos-x64@npm:0.25.12" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + +"@esbuild/win32-arm64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/win32-arm64@npm:0.25.12" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@esbuild/win32-ia32@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/win32-ia32@npm:0.25.12" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@esbuild/win32-x64@npm:0.25.12": + version: 0.25.12 + resolution: "@esbuild/win32-x64@npm:0.25.12" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@isaacs/fs-minipass@npm:^4.0.0": + version: 4.0.1 + resolution: "@isaacs/fs-minipass@npm:4.0.1" + dependencies: + minipass: "npm:^7.0.4" + checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.3" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.3" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.3" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.3" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.3" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3": + version: 3.0.3 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.3" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@types/node@npm:20": + version: 20.19.41 + resolution: "@types/node@npm:20.19.41" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/aa2a07317bbd700bea68d5784b403a738dbcebadbe2d8ef05649f7953065120d5d37f7edfdd7881df3a3bd15328c8a4dc46fdd69732ab540d552c505378c585b + languageName: node + linkType: hard + +"abbrev@npm:^4.0.0": + version: 4.0.0 + resolution: "abbrev@npm:4.0.0" + checksum: 10c0/b4cc16935235e80702fc90192e349e32f8ef0ed151ef506aa78c81a7c455ec18375c4125414b99f84b2e055199d66383e787675f0bcd87da7a4dbd59f9eac1d5 + languageName: node + linkType: hard + +"chownr@npm:^3.0.0": + version: 3.0.0 + resolution: "chownr@npm:3.0.0" + checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10 + languageName: node + linkType: hard + +"commander@npm:^12.1.0": + version: 12.1.0 + resolution: "commander@npm:12.1.0" + checksum: 10c0/6e1996680c083b3b897bfc1cfe1c58dfbcd9842fd43e1aaf8a795fbc237f65efcc860a3ef457b318e73f29a4f4a28f6403c3d653d021d960e4632dd45bde54a9 + languageName: node + linkType: hard + +"detect-libc@npm:^2.0.1": + version: 2.1.2 + resolution: "detect-libc@npm:2.1.2" + checksum: 10c0/acc675c29a5649fa1fb6e255f993b8ee829e510b6b56b0910666949c80c364738833417d0edb5f90e4e46be17228b0f2b66a010513984e18b15deeeac49369c4 + languageName: node + linkType: hard + +"env-paths@npm:^2.2.0": + version: 2.2.1 + resolution: "env-paths@npm:2.2.1" + checksum: 10c0/285325677bf00e30845e330eec32894f5105529db97496ee3f598478e50f008c5352a41a30e5e72ec9de8a542b5a570b85699cd63bd2bc646dbcb9f311d83bc4 + languageName: node + linkType: hard + +"esbuild@npm:^0.25.12": + version: 0.25.12 + resolution: "esbuild@npm:0.25.12" + dependencies: + "@esbuild/aix-ppc64": "npm:0.25.12" + "@esbuild/android-arm": "npm:0.25.12" + "@esbuild/android-arm64": "npm:0.25.12" + "@esbuild/android-x64": "npm:0.25.12" + "@esbuild/darwin-arm64": "npm:0.25.12" + "@esbuild/darwin-x64": "npm:0.25.12" + "@esbuild/freebsd-arm64": "npm:0.25.12" + "@esbuild/freebsd-x64": "npm:0.25.12" + "@esbuild/linux-arm": "npm:0.25.12" + "@esbuild/linux-arm64": "npm:0.25.12" + "@esbuild/linux-ia32": "npm:0.25.12" + "@esbuild/linux-loong64": "npm:0.25.12" + "@esbuild/linux-mips64el": "npm:0.25.12" + "@esbuild/linux-ppc64": "npm:0.25.12" + "@esbuild/linux-riscv64": "npm:0.25.12" + "@esbuild/linux-s390x": "npm:0.25.12" + "@esbuild/linux-x64": "npm:0.25.12" + "@esbuild/netbsd-arm64": "npm:0.25.12" + "@esbuild/netbsd-x64": "npm:0.25.12" + "@esbuild/openbsd-arm64": "npm:0.25.12" + "@esbuild/openbsd-x64": "npm:0.25.12" + "@esbuild/openharmony-arm64": "npm:0.25.12" + "@esbuild/sunos-x64": "npm:0.25.12" + "@esbuild/win32-arm64": "npm:0.25.12" + "@esbuild/win32-ia32": "npm:0.25.12" + "@esbuild/win32-x64": "npm:0.25.12" + dependenciesMeta: + "@esbuild/aix-ppc64": + optional: true + "@esbuild/android-arm": + optional: true + "@esbuild/android-arm64": + optional: true + "@esbuild/android-x64": + optional: true + "@esbuild/darwin-arm64": + optional: true + "@esbuild/darwin-x64": + optional: true + "@esbuild/freebsd-arm64": + optional: true + "@esbuild/freebsd-x64": + optional: true + "@esbuild/linux-arm": + optional: true + "@esbuild/linux-arm64": + optional: true + "@esbuild/linux-ia32": + optional: true + "@esbuild/linux-loong64": + optional: true + "@esbuild/linux-mips64el": + optional: true + "@esbuild/linux-ppc64": + optional: true + "@esbuild/linux-riscv64": + optional: true + "@esbuild/linux-s390x": + optional: true + "@esbuild/linux-x64": + optional: true + "@esbuild/netbsd-arm64": + optional: true + "@esbuild/netbsd-x64": + optional: true + "@esbuild/openbsd-arm64": + optional: true + "@esbuild/openbsd-x64": + optional: true + "@esbuild/openharmony-arm64": + optional: true + "@esbuild/sunos-x64": + optional: true + "@esbuild/win32-arm64": + optional: true + "@esbuild/win32-ia32": + optional: true + "@esbuild/win32-x64": + optional: true + bin: + esbuild: bin/esbuild + checksum: 10c0/c205357531423220a9de8e1e6c6514242bc9b1666e762cd67ccdf8fdfdc3f1d0bd76f8d9383958b97ad4c953efdb7b6e8c1f9ca5951cd2b7c5235e8755b34a6b + languageName: node + linkType: hard + +"exponential-backoff@npm:^3.1.1": + version: 3.1.3 + resolution: "exponential-backoff@npm:3.1.3" + checksum: 10c0/77e3ae682b7b1f4972f563c6dbcd2b0d54ac679e62d5d32f3e5085feba20483cf28bd505543f520e287a56d4d55a28d7874299941faf637e779a1aa5994d1267 + languageName: node + linkType: hard + +"fdir@npm:^6.5.0": + version: 6.5.0 + resolution: "fdir@npm:6.5.0" + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + checksum: 10c0/e345083c4306b3aed6cb8ec551e26c36bab5c511e99ea4576a16750ddc8d3240e63826cc624f5ae17ad4dc82e68a253213b60d556c11bfad064b7607847ed07f + languageName: node + linkType: hard + +"graceful-fs@npm:^4.2.6": + version: 4.2.11 + resolution: "graceful-fs@npm:4.2.11" + checksum: 10c0/386d011a553e02bc594ac2ca0bd6d9e4c22d7fa8cfbfc448a6d148c59ea881b092db9dbe3547ae4b88e55f1b01f7c4a2ecc53b310c042793e63aa44cf6c257f2 + languageName: node + linkType: hard + +"isexe@npm:^4.0.0": + version: 4.0.0 + resolution: "isexe@npm:4.0.0" + checksum: 10c0/5884815115bceac452877659a9c7726382531592f43dc29e5d48b7c4100661aed54018cb90bd36cb2eaeba521092570769167acbb95c18d39afdccbcca06c5ce + languageName: node + linkType: hard + +"minipass@npm:^7.0.4, minipass@npm:^7.1.2": + version: 7.1.3 + resolution: "minipass@npm:7.1.3" + checksum: 10c0/539da88daca16533211ea5a9ee98dc62ff5742f531f54640dd34429e621955e91cc280a91a776026264b7f9f6735947629f920944e9c1558369e8bf22eb33fbb + languageName: node + linkType: hard + +"minizlib@npm:^3.1.0": + version: 3.1.0 + resolution: "minizlib@npm:3.1.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/5aad75ab0090b8266069c9aabe582c021ae53eb33c6c691054a13a45db3b4f91a7fb1bd79151e6b4e9e9a86727b522527c0a06ec7d45206b745d54cd3097bcec + languageName: node + linkType: hard + +"msgpackr-extract@npm:^3.0.2": + version: 3.0.3 + resolution: "msgpackr-extract@npm:3.0.3" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.3" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.3" + node-gyp: "npm:latest" + node-gyp-build-optional-packages: "npm:5.2.2" + dependenciesMeta: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-darwin-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-win32-x64": + optional: true + bin: + download-msgpackr-prebuilds: bin/download-prebuilds.js + checksum: 10c0/e504fd8bf86a29d7527c83776530ee6dc92dcb0273bb3679fd4a85173efead7f0ee32fb82c8410a13c33ef32828c45f81118ffc0fbed5d6842e72299894623b4 + languageName: node + linkType: hard + +"msgpackr@npm:^1.11.2": + version: 1.11.12 + resolution: "msgpackr@npm:1.11.12" + dependencies: + msgpackr-extract: "npm:^3.0.2" + dependenciesMeta: + msgpackr-extract: + optional: true + checksum: 10c0/e9f1460e363dbd8c81a5c1b5829980edea7d76e91d570d094d0a4dae0d8ad12f64dea11b2be15f3d7b48d615fa9d3c9b600a6894fd272526087fa33753b5fd16 + languageName: node + linkType: hard + +"node-gyp-build-optional-packages@npm:5.2.2": + version: 5.2.2 + resolution: "node-gyp-build-optional-packages@npm:5.2.2" + dependencies: + detect-libc: "npm:^2.0.1" + bin: + node-gyp-build-optional-packages: bin.js + node-gyp-build-optional-packages-optional: optional.js + node-gyp-build-optional-packages-test: build-test.js + checksum: 10c0/c81128c6f91873381be178c5eddcbdf66a148a6a89a427ce2bcd457593ce69baf2a8662b6d22cac092d24aa9c43c230dec4e69b3a0da604503f4777cd77e282b + languageName: node + linkType: hard + +"node-gyp@npm:latest": + version: 12.3.0 + resolution: "node-gyp@npm:12.3.0" + dependencies: + env-paths: "npm:^2.2.0" + exponential-backoff: "npm:^3.1.1" + graceful-fs: "npm:^4.2.6" + nopt: "npm:^9.0.0" + proc-log: "npm:^6.0.0" + semver: "npm:^7.3.5" + tar: "npm:^7.5.4" + tinyglobby: "npm:^0.2.12" + undici: "npm:^6.25.0" + which: "npm:^6.0.0" + bin: + node-gyp: bin/node-gyp.js + checksum: 10c0/9d9032b405cbe42f72a105259d9eb679376470c102df4a2dbaa51e07d59bf741dcffb85897087ea9d8318b9cabb824a8978af51508ae142f0239ae1e6a3c2329 + languageName: node + linkType: hard + +"nopt@npm:^9.0.0": + version: 9.0.0 + resolution: "nopt@npm:9.0.0" + dependencies: + abbrev: "npm:^4.0.0" + bin: + nopt: bin/nopt.js + checksum: 10c0/1822eb6f9b020ef6f7a7516d7b64a8036e09666ea55ac40416c36e4b2b343122c3cff0e2f085675f53de1d2db99a2a89a60ccea1d120bcd6a5347bf6ceb4a7fd + languageName: node + linkType: hard + +"pako@npm:^2.1.0": + version: 2.1.0 + resolution: "pako@npm:2.1.0" + checksum: 10c0/8e8646581410654b50eb22a5dfd71159cae98145bd5086c9a7a816ec0370b5f72b4648d08674624b3870a521e6a3daffd6c2f7bc00fdefc7063c9d8232ff5116 + languageName: node + linkType: hard + +"picomatch@npm:^4.0.4": + version: 4.0.4 + resolution: "picomatch@npm:4.0.4" + checksum: 10c0/e2c6023372cc7b5764719a5ffb9da0f8e781212fa7ca4bd0562db929df8e117460f00dff3cb7509dacfc06b86de924b247f504d0ce1806a37fac4633081466b0 + languageName: node + linkType: hard + +"proc-log@npm:^6.0.0": + version: 6.1.0 + resolution: "proc-log@npm:6.1.0" + checksum: 10c0/4f178d4062733ead9d71a9b1ab24ebcecdfe2250916a5b1555f04fe2eda972a0ec76fbaa8df1ad9c02707add6749219d118a4fc46dc56bdfe4dde4b47d80bb82 + languageName: node + linkType: hard + +"semver@npm:^7.3.5": + version: 7.8.0 + resolution: "semver@npm:7.8.0" + bin: + semver: bin/semver.js + checksum: 10c0/8f096ca9b80ffd47b308d03f9ce8c873e27e2983f36023c559cdc92c51e8433fc23ebbfe57ec9623fc155636a6961ee989501099841ae4bb1babc8d2b3f048cd + languageName: node + linkType: hard + +"tar@npm:^7.5.4": + version: 7.5.15 + resolution: "tar@npm:7.5.15" + dependencies: + "@isaacs/fs-minipass": "npm:^4.0.0" + chownr: "npm:^3.0.0" + minipass: "npm:^7.1.2" + minizlib: "npm:^3.1.0" + yallist: "npm:^5.0.0" + checksum: 10c0/8f039edb1d12fdd7df6c6f9877d125afe9f3da3f5f9317df326fdd090d48793d6998cede1506a1471f3e3a250db270a89dace28005eb5e99c5a9132d704ac956 + languageName: node + linkType: hard + +"tinyglobby@npm:^0.2.12": + version: 0.2.16 + resolution: "tinyglobby@npm:0.2.16" + dependencies: + fdir: "npm:^6.5.0" + picomatch: "npm:^4.0.4" + checksum: 10c0/f2e09fd93dd95c41e522113b686ff6f7c13020962f8698a864a257f3d7737599afc47722b7ab726e12f8a813f779906187911ff8ee6701ede65072671a7e934b + languageName: node + linkType: hard + +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 + languageName: node + linkType: hard + +"undici@npm:^6.25.0": + version: 6.25.0 + resolution: "undici@npm:6.25.0" + checksum: 10c0/2597cc6689bdb02c210c557b1f85febbfda65becae6e6fc1061508e2f33734d25207f81cd8af56ada9956329eb3a7bd7431e87dcfeceba20ee87059b57dcf985 + languageName: node + linkType: hard + +"which@npm:^6.0.0": + version: 6.0.1 + resolution: "which@npm:6.0.1" + dependencies: + isexe: "npm:^4.0.0" + bin: + node-which: bin/which.js + checksum: 10c0/7e710e54ea36d2d6183bee2f9caa27a3b47b9baf8dee55a199b736fcf85eab3b9df7556fca3d02b50af7f3dfba5ea3a45644189836df06267df457e354da66d5 + languageName: node + linkType: hard + +"yallist@npm:^5.0.0": + version: 5.0.0 + resolution: "yallist@npm:5.0.0" + checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416 + languageName: node + linkType: hard From 4eb25eacd2a4f3d8d9e389cd7dfc69fc9540299c Mon Sep 17 00:00:00 2001 From: AztecBot Date: Fri, 22 May 2026 00:58:59 +0000 Subject: [PATCH 2/2] feat(wasm-bench): paired A/B mode with bootstrap CI, retry, and HTTP caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `chonk-ab` benchmark that alternates two wasm variants on a single BrowserStack worker, then reports per-variant medians + bootstrap 95% CI on the median Δ and a Wilcoxon signed-rank p-value, so a PR-vs-base statement is backed by a real significance test instead of a single-run point estimate. - `src/main.js`: paired A/B mode (`pairs`, `warmupPairs`, `variants`, `wasmBaseUrls`), balanced forward/reverse schedule, per-pair `pair_run` progress events, single consolidated `chonk-ab` result POSTed to `/result`. - `scripts/build-ab.mjs`: lay out two variants under `dest/wasm//` and write `variants.manifest.json` with md5s + an `allSameMd5` flag (used for the A==B harness validation run). - `scripts/analyze-ab.mjs`: per-variant `n / median / mean / stddev / min / max`, per-pair Δ (ms and %), seeded bootstrap 95% CI on median Δ, Wilcoxon signed-rank test, position-balance counter, and a sanity check that `proofFieldCount` and `verificationKeyBytes` match across variants. - `scripts/run-browserstack.mjs`: watchdog driver previously documented but missing from the tree (`firstProgressMs`, `stallMs`, `deadlineMs`, teardown-on-exit). - `src/wasm-runtime.js`: `mem.subarray → mem.slice` so `TextDecoder` accepts views over `SharedArrayBuffer` (Chrome / Safari hardened this; the WASM log-string path otherwise throws `must not be shared`); retry-with-backoff for the wasm fetch. - `src/crs.js`, `src/chonk.js`, `src/bench-worker.js`: retry+backoff on CRS fetches and per-stage progress events (`crs_fetch_start`, `crs_g1_fetched`, `srs_init_g1_start`, `srs_ready`, …) so a stalled paired worker is diagnosable. - `scripts/serve-bench.mjs`: serve `/wasm/...` and the CRS proxy with `Cache-Control: public, max-age=3600` so paired workers 2..N hit the browser HTTP cache instead of re-pulling 16 MB of G1 through the Cloudflare Quick Tunnel each time (the original cause of mid-run stalls). - `scripts/build.mjs`: single-thread wasm is now optional with a warning (paired A/B only needs the threaded build). - `scripts/test.mjs`: covers bootstrap-CI degeneracy, Wilcoxon edge cases, and a full `analyzeAB` round-trip on a synthetic A==B input. A==B null-result validation on BS macOS Chrome 148, N=10 analyzed pairs, same wasm at both `/wasm/a` and `/wasm/b` (md5 caea3a9477...): | metric | Δ% median | 95% CI | Wilcoxon p | |----------------|----------:|:--------------:|-----------:| | proveTotalMs | −0.062% | [−1.87, +1.63] | 0.92 | | setupMs | +0.112% | [−2.30, +1.42] | 0.84 | | proveMs | −0.139% | [−1.31, +1.89] | 1.00 | All CIs contain zero, Wilcoxon clearly not significant. --- barretenberg/wasm-bench/README.md | 54 +++ barretenberg/wasm-bench/package.json | 3 + .../wasm-bench/scripts/analyze-ab.mjs | 309 ++++++++++++++++++ barretenberg/wasm-bench/scripts/build-ab.mjs | 89 +++++ barretenberg/wasm-bench/scripts/build.mjs | 16 +- .../wasm-bench/scripts/run-browserstack.mjs | 246 ++++++++++++++ .../wasm-bench/scripts/serve-bench.mjs | 13 +- barretenberg/wasm-bench/scripts/test.mjs | 46 +++ barretenberg/wasm-bench/src/bench-worker.js | 8 +- barretenberg/wasm-bench/src/chonk.js | 3 + barretenberg/wasm-bench/src/crs.js | 32 +- barretenberg/wasm-bench/src/main.js | 267 ++++++++++++--- barretenberg/wasm-bench/src/wasm-runtime.js | 30 +- 13 files changed, 1038 insertions(+), 78 deletions(-) create mode 100644 barretenberg/wasm-bench/scripts/analyze-ab.mjs create mode 100644 barretenberg/wasm-bench/scripts/build-ab.mjs create mode 100644 barretenberg/wasm-bench/scripts/run-browserstack.mjs diff --git a/barretenberg/wasm-bench/README.md b/barretenberg/wasm-bench/README.md index 249fd9ee0f8d..8e23b80adec7 100644 --- a/barretenberg/wasm-bench/README.md +++ b/barretenberg/wasm-bench/README.md @@ -74,3 +74,57 @@ List supported targets: ```bash yarn create-link --target true ``` + +## Paired A/B (statistically meaningful PR vs base) + +For "is PR X faster than base?" comparisons, single-run timings on a shared +device are noise — between-run variance on a BrowserStack mobile/desktop +worker swamps anything under ~5%. The harness ships a paired A/B mode that +runs both variants on the same physical worker in alternating order: + +1. Build the wasm for both sides (e.g. `barretenberg.wasm.gz` for PR head and + merge-base). Either of `cmake --preset wasm-threads -DENABLE_WASM_BENCH=ON` + variants is fine — just two separate output files. +2. After `yarn build`, lay them both out under `dest/wasm//`: + + ```bash + yarn build-ab \ + --variant pr=path/to/pr/barretenberg.wasm.gz \ + --variant base=path/to/base/barretenberg.wasm.gz + ``` + + For an A==B harness validation run, pass the same wasm for both variants; + the generated `variants.manifest.json` reports identical md5s and the + ground-truth Δ should bracket zero. +3. Drive the worker with `bench` params: + + ```json + { + "benchmark": "chonk-ab", + "flow": "ecdsar1+transfer_1_recursions+sponsored_fpc", + "threads": "auto", + "pairs": 11, + "warmupPairs": 1, + "variants": ["pr", "base"], + "wasmBaseUrls": { "pr": "/wasm/pr", "base": "/wasm/base" } + } + ``` + + The first pair is dropped from analysis (caches/JIT warm-up). Order + alternates every pair so PR-first and base-first counts are balanced. +4. Post-collection, run the analyzer: + + ```bash + yarn analyze-ab --result /tmp/wasm-bench-results.jsonl + ``` + + It reports per-variant `n / median / mean / stddev / min / max`, per-pair + Δ summary, seeded bootstrap 95% CI on the median Δ (both ms and %), and + a Wilcoxon signed-rank test on the paired deltas. Verdict is + `significant` if and only if the Δ% 95% CI excludes zero. + +`scripts/run-browserstack.mjs` drives the whole loop with watchdogs +(`--stall-ms`, `--deadline-ms`, per-target `firstProgressMs`), when run from +a host that has `BROWSERSTACK_USERNAME` / `BROWSERSTACK_ACCESS_KEY` in env. +From a claudebox session, use the BrowserStack MCP tools instead — the +`bench` param shape is the same. diff --git a/barretenberg/wasm-bench/package.json b/barretenberg/wasm-bench/package.json index 47c33307b097..b0330db66af5 100644 --- a/barretenberg/wasm-bench/package.json +++ b/barretenberg/wasm-bench/package.json @@ -5,7 +5,10 @@ "type": "module", "scripts": { "build": "node scripts/build.mjs", + "build-ab": "node scripts/build-ab.mjs", + "analyze-ab": "node scripts/analyze-ab.mjs", "create-link": "node scripts/create-link.mjs", + "run-browserstack": "node scripts/run-browserstack.mjs", "serve": "node scripts/serve-bench.mjs", "test": "node scripts/test.mjs" }, diff --git a/barretenberg/wasm-bench/scripts/analyze-ab.mjs b/barretenberg/wasm-bench/scripts/analyze-ab.mjs new file mode 100644 index 000000000000..1c5b38c60295 --- /dev/null +++ b/barretenberg/wasm-bench/scripts/analyze-ab.mjs @@ -0,0 +1,309 @@ +#!/usr/bin/env node +// Analyze a wasm-bench paired A/B result file. +// +// Reads either: +// - a result JSONL produced by serve-bench.mjs (--result-jsonl path), or +// - a single JSON file containing one `chonk-ab` result object. +// +// Reports per-variant {n, mean, median, stddev, min, max} and per-pair +// Δ = proveTotal[variants[0]] − proveTotal[variants[1]] (and Δ%) with a +// seeded bootstrap 95% CI on the median Δ, plus a Wilcoxon signed-rank +// statistic on the paired Δ. Asserts proofFieldCount and +// verificationKeyBytes are identical across pairs/variants (catches +// "we benchmarked different circuits" bugs). + +import { readFileSync } from 'node:fs'; +import { Command } from 'commander'; + +function mean(values) { + if (values.length === 0) return NaN; + let s = 0; + for (const v of values) s += v; + return s / values.length; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const n = sorted.length; + if (n === 0) return NaN; + return n % 2 === 1 ? sorted[(n - 1) / 2] : 0.5 * (sorted[n / 2 - 1] + sorted[n / 2]); +} + +function stddev(values) { + const n = values.length; + if (n < 2) return NaN; + const m = mean(values); + let ss = 0; + for (const v of values) ss += (v - m) * (v - m); + return Math.sqrt(ss / (n - 1)); +} + +function summarize(values) { + return { + n: values.length, + mean: mean(values), + median: median(values), + stddev: stddev(values), + min: Math.min(...values), + max: Math.max(...values), + }; +} + +function mulberry32(seed) { + let s = seed >>> 0; + return function () { + s = (s + 0x6D2B79F5) >>> 0; + let t = s; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export function bootstrapMedianCI(values, { iters = 4000, alpha = 0.05, seed = 42 } = {}) { + const n = values.length; + if (n === 0) return { lo: NaN, hi: NaN, point: NaN, n, iters }; + const rng = mulberry32(seed); + const meds = new Float64Array(iters); + for (let i = 0; i < iters; i++) { + const sample = new Array(n); + for (let j = 0; j < n; j++) sample[j] = values[Math.floor(rng() * n)]; + meds[i] = median(sample); + } + const sorted = Array.from(meds).sort((a, b) => a - b); + const loIdx = Math.max(0, Math.floor((alpha / 2) * iters)); + const hiIdx = Math.min(iters - 1, Math.ceil((1 - alpha / 2) * iters) - 1); + return { lo: sorted[loIdx], hi: sorted[hiIdx], point: median(values), n, iters, seed }; +} + +function erf(x) { + const sign = x >= 0 ? 1 : -1; + const a1 = 0.254829592; + const a2 = -0.284496736; + const a3 = 1.421413741; + const a4 = -1.453152027; + const a5 = 1.061405429; + const p = 0.3275911; + const t = 1.0 / (1 + p * Math.abs(x)); + const y = 1.0 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-x * x); + return sign * y; +} + +function normalCdf(z) { + return 0.5 * (1 + erf(z / Math.SQRT2)); +} + +export function wilcoxonSignedRank(deltas) { + const nonZero = deltas.filter(d => d !== 0); + const n = nonZero.length; + if (n === 0) return { n, wPlus: 0, wMinus: 0, statistic: NaN, z: NaN, p: NaN }; + const abs = nonZero.map(d => ({ abs: Math.abs(d), sign: d > 0 ? 1 : -1 })); + abs.sort((a, b) => a.abs - b.abs); + let i = 0; + while (i < n) { + let j = i; + while (j < n - 1 && abs[j + 1].abs === abs[i].abs) j++; + const avgRank = (i + j) / 2 + 1; + for (let k = i; k <= j; k++) abs[k].rank = avgRank; + i = j + 1; + } + let wPlus = 0; + let wMinus = 0; + for (const x of abs) { + if (x.sign > 0) wPlus += x.rank; + else wMinus += x.rank; + } + const W = Math.min(wPlus, wMinus); + const mu = (n * (n + 1)) / 4; + const sigma = Math.sqrt((n * (n + 1) * (2 * n + 1)) / 24); + const z = sigma === 0 ? 0 : (W - mu + 0.5) / sigma; + const p = 2 * normalCdf(-Math.abs(z)); + return { n, wPlus, wMinus, statistic: W, z, p }; +} + +function loadResult(path) { + const raw = readFileSync(path, 'utf8').trim(); + if (raw.startsWith('{')) { + return JSON.parse(raw); + } + // JSONL — take the last `chonk-ab` row. + let last; + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + if (parsed.benchmark === 'chonk-ab') last = parsed; + } + if (!last) { + throw new Error(`No chonk-ab row found in ${path}`); + } + return last; +} + +function fmtPct(value) { + if (!Number.isFinite(value)) return 'n/a'; + return `${value >= 0 ? '+' : ''}${value.toFixed(3)}%`; +} + +function fmtMs(value) { + if (!Number.isFinite(value)) return 'n/a'; + return `${value.toFixed(1)} ms`; +} + +function metricFromRun(run, metric) { + if (!run) return NaN; + if (metric === 'proveTotalMs') { + if (Number.isFinite(run.proveTotalMs)) return run.proveTotalMs; + return Number(run.setupMs ?? 0) + Number(run.proveMs ?? 0); + } + return Number(run[metric]); +} + +export function analyzeAB(result, { warmupPairs = result.warmupPairs ?? 1, metrics = ['proveTotalMs', 'setupMs', 'proveMs'], bootstrapIters = 4000, seed = 42 } = {}) { + const variants = result.variants; + if (!Array.isArray(variants) || variants.length !== 2) { + throw new Error(`Result.variants must be a 2-element array; got ${JSON.stringify(variants)}`); + } + const [varA, varB] = variants; + + // Group by pair index. + const byPair = new Map(); + for (const entry of result.pairs) { + if (!byPair.has(entry.pair)) byPair.set(entry.pair, {}); + byPair.get(entry.pair)[entry.variant] = entry; + } + const sortedPairs = [...byPair.keys()].sort((a, b) => a - b); + + // Apply warmup drop. + const analyzedPairs = sortedPairs.filter(p => p >= warmupPairs); + const aRuns = analyzedPairs.map(p => byPair.get(p)?.[varA]); + const bRuns = analyzedPairs.map(p => byPair.get(p)?.[varB]); + + // Sanity: same circuit across all runs. + const proofFieldCounts = new Set([...aRuns, ...bRuns].map(e => e?.run?.proofFieldCount)); + const vkBytes = new Set([...aRuns, ...bRuns].map(e => e?.run?.verificationKeyBytes)); + const allVerified = [...aRuns, ...bRuns].every(e => e?.run?.verified === true); + + const sanity = { + proofFieldCounts: [...proofFieldCounts], + verificationKeyBytes: [...vkBytes], + allVerified, + }; + + // Build position-balance tally (how many pair entries had A first vs B first). + const positionBalance = { aFirst: 0, bFirst: 0 }; + for (const p of analyzedPairs) { + const entry = byPair.get(p); + if (!entry) continue; + if ((entry[varA]?.position ?? -1) < (entry[varB]?.position ?? Infinity)) positionBalance.aFirst++; + else positionBalance.bFirst++; + } + + const out = { + variants, + flow: result.flow, + pairs: sortedPairs.length, + warmupPairs, + analyzedPairs: analyzedPairs.length, + positionBalance, + sanity, + perMetric: {}, + }; + + for (const metric of metrics) { + const aVals = aRuns.map(e => metricFromRun(e?.run, metric)); + const bVals = bRuns.map(e => metricFromRun(e?.run, metric)); + const validIdx = aVals.map((_, i) => i).filter(i => Number.isFinite(aVals[i]) && Number.isFinite(bVals[i])); + const aPaired = validIdx.map(i => aVals[i]); + const bPaired = validIdx.map(i => bVals[i]); + const deltas = validIdx.map(i => aVals[i] - bVals[i]); + const deltaPcts = validIdx.map(i => (100 * (aVals[i] - bVals[i])) / bVals[i]); + + const aSummary = summarize(aPaired); + const bSummary = summarize(bPaired); + const deltaSummary = summarize(deltas); + const deltaPctSummary = summarize(deltaPcts); + const ciMs = bootstrapMedianCI(deltas, { iters: bootstrapIters, seed }); + const ciPct = bootstrapMedianCI(deltaPcts, { iters: bootstrapIters, seed: seed + 1 }); + const wilcoxon = wilcoxonSignedRank(deltas); + + const significant = ciPct.lo > 0 || ciPct.hi < 0; + + out.perMetric[metric] = { + a: { variant: varA, ...aSummary }, + b: { variant: varB, ...bSummary }, + deltaMs: { ...deltaSummary, ci95: { lo: ciMs.lo, hi: ciMs.hi, point: ciMs.point } }, + deltaPct: { ...deltaPctSummary, ci95: { lo: ciPct.lo, hi: ciPct.hi, point: ciPct.point } }, + wilcoxon, + significant, + }; + } + + return out; +} + +function renderReport(analysis) { + const lines = []; + lines.push(`# wasm-bench A/B analysis`); + lines.push(''); + lines.push(`Flow: ${analysis.flow}`); + lines.push(`Variants: ${analysis.variants[0]} vs ${analysis.variants[1]}`); + lines.push(`Pairs total: ${analysis.pairs}; warmup dropped: ${analysis.warmupPairs}; analyzed: ${analysis.analyzedPairs}`); + lines.push(`Position balance (A first / B first): ${analysis.positionBalance.aFirst} / ${analysis.positionBalance.bFirst}`); + lines.push(`Sanity — proofFieldCount(s): ${analysis.sanity.proofFieldCounts.join(', ')}; vkBytes: ${analysis.sanity.verificationKeyBytes.join(', ')}; allVerified: ${analysis.sanity.allVerified}`); + lines.push(''); + for (const [metric, m] of Object.entries(analysis.perMetric)) { + lines.push(`## ${metric}`); + lines.push(''); + lines.push(`| Side | n | median | mean | stddev | min | max |`); + lines.push(`|---|---:|---:|---:|---:|---:|---:|`); + lines.push(`| ${m.a.variant} | ${m.a.n} | ${fmtMs(m.a.median)} | ${fmtMs(m.a.mean)} | ${fmtMs(m.a.stddev)} | ${fmtMs(m.a.min)} | ${fmtMs(m.a.max)} |`); + lines.push(`| ${m.b.variant} | ${m.b.n} | ${fmtMs(m.b.median)} | ${fmtMs(m.b.mean)} | ${fmtMs(m.b.stddev)} | ${fmtMs(m.b.min)} | ${fmtMs(m.b.max)} |`); + lines.push(''); + lines.push(`Δ (${m.a.variant} − ${m.b.variant}):`); + lines.push(`- Δ median = ${fmtMs(m.deltaMs.median)} (95% CI [${fmtMs(m.deltaMs.ci95.lo)}, ${fmtMs(m.deltaMs.ci95.hi)}])`); + lines.push(`- Δ% median = ${fmtPct(m.deltaPct.median)} (95% CI [${fmtPct(m.deltaPct.ci95.lo)}, ${fmtPct(m.deltaPct.ci95.hi)}])`); + lines.push(`- Δ stddev = ${fmtMs(m.deltaMs.stddev)}; min/max = ${fmtMs(m.deltaMs.min)} / ${fmtMs(m.deltaMs.max)}`); + lines.push(`- Wilcoxon signed-rank: W=${m.wilcoxon.statistic}, z=${m.wilcoxon.z?.toFixed(3)}, p≈${m.wilcoxon.p?.toFixed(4)}`); + lines.push(`- Verdict: ${m.significant ? 'Δ% 95% CI excludes zero — significant' : 'Δ% 95% CI contains zero — not distinguishable from zero at this N'}`); + lines.push(''); + } + return lines.join('\n'); +} + +async function main(argv) { + const program = new Command(); + program + .option('--result ', 'Path to chonk-ab JSON or JSONL result file', '/tmp/wasm-bench-results.jsonl') + .option('--warmup ', 'Number of warmup pairs to drop (defaults to result.warmupPairs)', value => Number.parseInt(value, 10)) + .option('--metric ', 'Metrics to analyze', value => value.split(','), ['proveTotalMs', 'setupMs', 'proveMs']) + .option('--bootstrap-iters ', 'Bootstrap iterations', value => Number.parseInt(value, 10), 4000) + .option('--seed ', 'PRNG seed', value => Number.parseInt(value, 10), 42) + .option('--json', 'Emit JSON analysis instead of markdown') + .parse(argv); + + const options = program.opts(); + const result = loadResult(options.result); + const analysis = analyzeAB(result, { + warmupPairs: options.warmup, + metrics: options.metric.flatMap(m => m.split(',').map(s => s.trim()).filter(Boolean)), + bootstrapIters: options.bootstrapIters, + seed: options.seed, + }); + if (options.json) { + process.stdout.write(`${JSON.stringify(analysis, null, 2)}\n`); + } else { + process.stdout.write(`${renderReport(analysis)}\n`); + } +} + +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + main(process.argv).catch(error => { + console.error(error.stack || error.message || error); + process.exitCode = 1; + }); +} diff --git a/barretenberg/wasm-bench/scripts/build-ab.mjs b/barretenberg/wasm-bench/scripts/build-ab.mjs new file mode 100644 index 000000000000..4da17c725e5b --- /dev/null +++ b/barretenberg/wasm-bench/scripts/build-ab.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node +// Lay out two-variant wasm bundles for paired A/B runs. +// +// Use after `node scripts/build.mjs` has produced dest/. Copies the chosen +// wasm.gz files into dest/wasm//barretenberg-threads.wasm.gz and +// records md5sums (and an optional "same-wasm" flag for A==B harness sanity). +// +// Usage: +// node scripts/build-ab.mjs \ +// --variant a=build-wasm-threads/bin/barretenberg.wasm.gz \ +// --variant b=build-wasm-threads/bin/barretenberg.wasm.gz +// +// For A==B harness validation, point both variants at the same file. The +// resulting manifest reports identical md5s, which downstream analyzers can +// use to assert "ground truth Δ should be zero". + +import { copyFile, mkdir, readFile, writeFile, stat } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { resolve } from 'node:path'; + +import { Command } from 'commander'; + +import { defaultDistDir } from './lib.mjs'; + +async function md5(path) { + const data = await readFile(path); + return createHash('md5').update(data).digest('hex'); +} + +async function main(argv) { + const program = new Command(); + program + .option('--dist ', 'Built harness directory', process.env.WASM_BENCH_DIST_DIR || defaultDistDir) + .option('--variant ', 'Variant "name=path" pair (repeatable)', collect, []) + .option('--single-variant-name ', 'When set, alias the legacy /wasm/ layout to this variant name as well') + .parse(argv); + + const options = program.opts(); + if (!options.variant?.length) { + throw new Error('At least one --variant name=path is required'); + } + + const dist = resolve(options.dist); + const manifest = { + generatedAt: new Date().toISOString(), + dist, + variants: {}, + }; + + for (const spec of options.variant) { + const idx = spec.indexOf('='); + if (idx <= 0) { + throw new Error(`Malformed --variant "${spec}"; expected name=path`); + } + const name = spec.slice(0, idx).trim(); + if (!/^[a-z0-9_-]+$/i.test(name)) { + throw new Error(`Invalid variant name "${name}"; allow [A-Za-z0-9_-] only`); + } + const source = resolve(spec.slice(idx + 1)); + const info = await stat(source); + if (!info.isFile() || info.size === 0) { + throw new Error(`Source wasm not readable or empty: ${source}`); + } + const variantDir = resolve(dist, 'wasm', name); + const target = resolve(variantDir, 'barretenberg-threads.wasm.gz'); + await mkdir(variantDir, { recursive: true }); + await copyFile(source, target); + const hash = await md5(target); + manifest.variants[name] = { source, target, sizeBytes: info.size, md5: hash }; + console.log(`variant ${name}: ${source} → ${target} (${info.size} B, md5=${hash})`); + } + + const md5s = Object.values(manifest.variants).map(v => v.md5); + manifest.allSameMd5 = md5s.every(h => h === md5s[0]); + manifest.note = manifest.allSameMd5 + ? 'All variant md5s are identical — A==B harness validation (ground truth Δ should be zero).' + : 'Variant md5s differ — A/B comparison harness.'; + await writeFile(resolve(dist, 'wasm', 'variants.manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`); + console.log(manifest.note); +} + +function collect(value, previous) { + return previous ? [...previous, value] : [value]; +} + +main(process.argv).catch(error => { + console.error(error.message ?? error); + process.exitCode = 1; +}); diff --git a/barretenberg/wasm-bench/scripts/build.mjs b/barretenberg/wasm-bench/scripts/build.mjs index 6105c60eabd9..ceadbf95b975 100644 --- a/barretenberg/wasm-bench/scripts/build.mjs +++ b/barretenberg/wasm-bench/scripts/build.mjs @@ -6,28 +6,36 @@ import { build } from 'esbuild'; import { defaultDistDir, packageRoot, repoRoot } from './lib.mjs'; const distDir = resolve(process.env.WASM_BENCH_DIST_DIR || defaultDistDir); +const allowMissing = process.env.WASM_BENCH_ALLOW_MISSING_WASM === '1'; const wasmOutputs = [ { source: resolve(repoRoot, 'barretenberg/cpp/build-wasm/bin/barretenberg.wasm.gz'), dest: 'barretenberg.wasm.gz', label: 'single-thread wasm', + optional: true, command: 'cd barretenberg/cpp && cmake --preset wasm -DENABLE_WASM_BENCH=ON && cmake --build --preset wasm --target barretenberg.wasm.gz', }, { source: resolve(repoRoot, 'barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm.gz'), dest: 'barretenberg-threads.wasm.gz', label: 'threaded wasm', + optional: false, command: 'cd barretenberg/cpp && cmake --preset wasm-threads && cmake --build --preset wasm-threads --target barretenberg.wasm.gz', }, ]; -async function assertReadable(path, label, command) { +async function assertReadable(path, label, command, { optional = false } = {}) { try { const info = await stat(path); if (!info.isFile() || info.size === 0) { throw new Error(`${label} is empty`); } + return true; } catch (error) { + if (optional || allowMissing) { + console.warn(`warning: ${label} not found at ${path} — skipping`); + return false; + } throw new Error(`${label} not found at ${path}.\nBuild it first:\n ${command}`, { cause: error }); } } @@ -53,8 +61,10 @@ await build({ await copyFile(resolve(packageRoot, 'src/index.html'), resolve(distDir, 'index.html')); for (const output of wasmOutputs) { - await assertReadable(output.source, output.label, output.command); - await copyFile(output.source, resolve(distDir, 'wasm', output.dest)); + const present = await assertReadable(output.source, output.label, output.command, { optional: output.optional }); + if (present) { + await copyFile(output.source, resolve(distDir, 'wasm', output.dest)); + } } console.log(`Built wasm bench into ${distDir}`); diff --git a/barretenberg/wasm-bench/scripts/run-browserstack.mjs b/barretenberg/wasm-bench/scripts/run-browserstack.mjs new file mode 100644 index 000000000000..7e90f01ea29a --- /dev/null +++ b/barretenberg/wasm-bench/scripts/run-browserstack.mjs @@ -0,0 +1,246 @@ +#!/usr/bin/env node +// Drive a BrowserStack JS Testing worker against a wasm-bench tunnel, tail the +// progress JSONL, and fail fast on stall / deadline / no-first-progress. +// +// Requires BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY in the environment +// (the agent shell does NOT have these — drive via the BrowserStack MCP tools +// from a claudebox session, or run this script from a host that has them). +// +// On success, prints a `==== headline ====` block with proveTotalMs (or, for +// paired A/B runs, per-variant medians and Δ summary). + +import { stat, watch } from 'node:fs/promises'; +import { readFileSync, statSync } from 'node:fs'; +import { Buffer } from 'node:buffer'; +import { Command } from 'commander'; + +import { + createBenchOptions, + createBrowserStackWorkerBody, + getTarget, + loadConfig, + proveTotalMs, + withBenchParam, +} from './lib.mjs'; + +const BS_API = 'https://api.browserstack.com/5'; + +function basicAuth(user, key) { + return `Basic ${Buffer.from(`${user}:${key}`).toString('base64')}`; +} + +async function bsCall(path, { method = 'GET', body, user, key }) { + const init = { + method, + headers: { + Authorization: basicAuth(user, key), + ...(body ? { 'Content-Type': 'application/json' } : {}), + }, + }; + if (body) init.body = JSON.stringify(body); + const response = await fetch(`${BS_API}${path}`, init); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`BrowserStack ${method} ${path} → ${response.status}: ${text.slice(0, 500)}`); + } + return response.json(); +} + +async function readProgressTail(path, fromOffset) { + try { + const info = await stat(path); + if (info.size <= fromOffset) return { rows: [], nextOffset: fromOffset }; + const buffer = Buffer.alloc(info.size - fromOffset); + const { open } = await import('node:fs/promises'); + const fh = await open(path, 'r'); + try { + await fh.read(buffer, 0, buffer.byteLength, fromOffset); + } finally { + await fh.close(); + } + const rows = []; + for (const line of buffer.toString('utf8').split('\n')) { + if (!line.trim()) continue; + try { + rows.push(JSON.parse(line)); + } catch { + // skip malformed lines + } + } + return { rows, nextOffset: info.size }; + } catch { + return { rows: [], nextOffset: fromOffset }; + } +} + +async function deleteWorker(workerId, user, key) { + if (!workerId) return; + try { + await bsCall(`/worker/${workerId}`, { method: 'DELETE', user, key }); + } catch (error) { + console.error(`worker teardown failed: ${error.message}`); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function main(argv) { + const program = new Command(); + program + .requiredOption('--url ', 'Public origin where wasm-bench is served (e.g. Cloudflare Quick Tunnel)') + .option('--target ', 'Hardware-target preset', 'macos') + .option('--flow ', 'Chonk flow name') + .option('--runs ', 'Runs per variant (single-variant mode)', value => Number.parseInt(value, 10)) + .option('--pairs ', 'Number of paired A/B pairs (A/B mode)', value => Number.parseInt(value, 10)) + .option('--variants ', 'Comma-separated variants (A/B mode), default "a,b"', 'a,b') + .option('--warmup-pairs ', 'Warmup pairs to drop in A/B analysis', value => Number.parseInt(value, 10), 1) + .option('--threads ', 'Threads per worker') + .option('--mem-max-pages ', 'WebAssembly.Memory max pages', value => Number.parseInt(value, 10)) + .option('--trace ', 'Capture Perfetto trace') + .option('--progress-jsonl ', 'Progress JSONL path tailed by the watchdog', '/tmp/wasm-bench-progress.jsonl') + .option('--first-progress-ms ', 'Max wait for first /progress event', value => Number.parseInt(value, 10)) + .option('--stall-ms ', 'Max gap between /progress events once started', value => Number.parseInt(value, 10), 180000) + .option('--deadline-ms ', 'Total wall-clock deadline', value => Number.parseInt(value, 10), 1800000) + .parse(argv); + + const options = program.opts(); + const user = process.env.BROWSERSTACK_USERNAME; + const key = process.env.BROWSERSTACK_ACCESS_KEY; + if (!user || !key) { + throw new Error('BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY not set in environment'); + } + + const config = loadConfig(); + const target = getTarget(config, options.target); + const firstProgressMs = options.firstProgressMs ?? target.firstProgressMs ?? 60000; + + // Build the bench options. A/B mode if --pairs is set. + const variantList = options.variants.split(',').map(s => s.trim()).filter(Boolean); + let benchOptions; + if (options.pairs && options.pairs > 0) { + benchOptions = { + benchmark: 'chonk-ab', + flow: options.flow ?? config.defaultFlow, + threads: options.threads ?? config.defaultThreads, + pairs: options.pairs, + warmupPairs: options.warmupPairs ?? 1, + variants: variantList, + wasmBaseUrls: Object.fromEntries(variantList.map(v => [v, `/wasm/${v}`])), + ...(options.memMaxPages ? { memMaxPages: options.memMaxPages } : {}), + ...(target.benchOverrides ?? {}), + }; + } else { + benchOptions = createBenchOptions(config, options.target, { + flow: options.flow, + runs: options.runs, + threads: options.threads, + memMaxPages: options.memMaxPages, + }); + if (options.trace !== undefined) { + benchOptions.trace = options.trace === 'true' || options.trace === true; + } + } + + const benchUrl = withBenchParam(options.url, benchOptions); + const workerBody = createBrowserStackWorkerBody(options.target, target, benchUrl, { + timeout: Math.max(1800, Math.floor(options.deadlineMs / 1000)), + }); + if (!workerBody) { + throw new Error(`Target "${options.target}" has no browserstackWorker config`); + } + + console.log(`bench url: ${benchUrl}`); + console.log(`spawning BrowserStack worker (target=${options.target}, mode=${benchOptions.benchmark})`); + const startWall = Date.now(); + const createResp = await bsCall('/worker', { method: 'POST', body: workerBody, user, key }); + const workerId = createResp.id; + console.log(`worker id=${workerId} browser_url=${createResp.browser_url ?? 'pending'}`); + + let progressOffset = 0; + let firstProgressSeen = false; + let lastProgressAt = Date.now(); + let lastSummary = null; + let abFinalRow = null; + + const cleanup = async () => { + try { await deleteWorker(workerId, user, key); } catch {} + }; + process.on('SIGINT', async () => { await cleanup(); process.exit(130); }); + + try { + while (true) { + const elapsed = Date.now() - startWall; + if (elapsed > options.deadlineMs) { + throw new Error(`deadline exceeded after ${(elapsed / 1000).toFixed(1)}s (no completion)`); + } + const { rows, nextOffset } = await readProgressTail(options.progressJsonl, progressOffset); + progressOffset = nextOffset; + if (rows.length > 0) { + if (!firstProgressSeen) { + firstProgressSeen = true; + console.log(`first /progress at +${((Date.now() - startWall) / 1000).toFixed(1)}s`); + } + lastProgressAt = Date.now(); + for (const row of rows) { + if (row.type === 'pair_run') { + console.log(`pair=${row.pair} variant=${row.variant} pos=${row.position} proveTotalMs=${Math.round(row.proveTotalMs ?? 0)} setupMs=${Math.round(row.setupMs ?? 0)} proveMs=${Math.round(row.proveMs ?? 0)}`); + } else if (row.type === 'run_complete') { + lastSummary = row; + console.log(`run_complete proveTotalMs=${Math.round(proveTotalMs(row.data ?? row))}`); + } else if (row.event === 'ab_complete') { + abFinalRow = row; + } else if (row.type === 'error' || row.type === 'ab_error') { + throw new Error(`browser-side error: ${row.error?.message ?? JSON.stringify(row.error)}`); + } + } + } + if (!firstProgressSeen && elapsed > firstProgressMs) { + throw new Error(`no /progress within ${firstProgressMs}ms — BrowserStack page likely never loaded`); + } + if (firstProgressSeen) { + const stalled = Date.now() - lastProgressAt; + if (stalled > options.stallMs) { + throw new Error(`progress stalled for ${(stalled / 1000).toFixed(1)}s (>${options.stallMs}ms)`); + } + } + // Poll worker status — if it's offline (BS terminated session), bail out. + try { + const status = await bsCall(`/worker/${workerId}/status`, { user, key }); + if (status.status === 'offline' || status.status === 'terminated') { + console.log(`worker reached terminal status=${status.status}`); + break; + } + } catch (error) { + console.error(`status poll error: ${error.message}`); + } + if (abFinalRow) { + console.log(`ab_complete row seen — finishing`); + break; + } + await sleep(3000); + } + + console.log('==== headline ===='); + if (benchOptions.benchmark === 'chonk-ab') { + console.log(`A/B run: pairs=${benchOptions.pairs} variants=${variantList.join(',')} flow=${benchOptions.flow}`); + console.log('Run analyze-ab to compute medians, CI, and significance:'); + console.log(` node scripts/analyze-ab.mjs --result --warmup ${benchOptions.warmupPairs ?? 1}`); + } else if (lastSummary) { + const data = lastSummary.data ?? lastSummary; + console.log(`proveTotalMs=${Math.round(proveTotalMs(data))} setupMs=${Math.round(data.setupMs ?? 0)} proveMs=${Math.round(data.proveMs ?? 0)} flow=${benchOptions.flow}`); + } else { + console.log('no completion summary captured'); + } + } finally { + await cleanup(); + } +} + +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) { + main(process.argv).catch(error => { + console.error(error.stack || error.message || error); + process.exitCode = 1; + }); +} diff --git a/barretenberg/wasm-bench/scripts/serve-bench.mjs b/barretenberg/wasm-bench/scripts/serve-bench.mjs index 7c65559cb78a..990900dae697 100644 --- a/barretenberg/wasm-bench/scripts/serve-bench.mjs +++ b/barretenberg/wasm-bench/scripts/serve-bench.mjs @@ -27,15 +27,15 @@ const contentTypes = new Map([ ['.gz', 'application/gzip'], ]); -function setSharedHeaders(response) { +function setSharedHeaders(response, { cache = 'no-store' } = {}) { response.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); response.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); response.setHeader('Cross-Origin-Resource-Policy', 'same-origin'); - response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Cache-Control', cache); } -function send(response, status, body, headers = {}) { - setSharedHeaders(response); +function send(response, status, body, headers = {}, sharedOpts = {}) { + setSharedHeaders(response, sharedOpts); response.writeHead(status, headers); response.end(body); } @@ -113,7 +113,7 @@ async function proxyCrs(response, pathname, searchParams) { send(response, 200, bytes, { 'Content-Type': 'application/octet-stream', 'Content-Length': String(bytes.byteLength), - }); + }, { cache: 'public, max-age=3600' }); return true; } @@ -140,7 +140,8 @@ async function serveStatic(response, root, pathname) { send(response, 404, 'not found', { 'Content-Type': 'text/plain; charset=utf-8' }); return; } - setSharedHeaders(response); + const isWasm = filePath.endsWith('.wasm.gz') || filePath.endsWith('.wasm'); + setSharedHeaders(response, isWasm ? { cache: 'public, max-age=3600' } : {}); response.writeHead(200, { 'Content-Type': contentTypes.get(extname(filePath)) ?? contentTypes.get(extname(basename(filePath, '.gz'))) ?? 'application/octet-stream', 'Content-Length': String(info.size), diff --git a/barretenberg/wasm-bench/scripts/test.mjs b/barretenberg/wasm-bench/scripts/test.mjs index f58bd617c00d..a6222c106a93 100644 --- a/barretenberg/wasm-bench/scripts/test.mjs +++ b/barretenberg/wasm-bench/scripts/test.mjs @@ -17,6 +17,7 @@ import { safeResolve, } from './lib.mjs'; import { createBenchServer } from './serve-bench.mjs'; +import { analyzeAB, bootstrapMedianCI, wilcoxonSignedRank } from './analyze-ab.mjs'; test('base64url bench parameters round-trip JSON', () => { const value = { @@ -102,6 +103,51 @@ test('html preview URL supports query-style services', () => { ); }); +test('bootstrap median CI on constant data is degenerate at the constant', () => { + const ci = bootstrapMedianCI([5, 5, 5, 5, 5, 5, 5, 5], { iters: 500, seed: 1 }); + assert.equal(ci.point, 5); + assert.equal(ci.lo, 5); + assert.equal(ci.hi, 5); +}); + +test('bootstrap median CI brackets the true median for symmetric noise', () => { + // Symmetric around 0 → median CI should bracket 0 at 95%. + const samples = [-3, -2, -1, 0, 0, 1, 2, 3]; + const ci = bootstrapMedianCI(samples, { iters: 2000, seed: 42 }); + assert.ok(ci.lo <= 0 && ci.hi >= 0, `CI [${ci.lo}, ${ci.hi}] should bracket 0`); +}); + +test('wilcoxon signed-rank returns NaN p-value when all deltas are zero', () => { + const { p } = wilcoxonSignedRank([0, 0, 0, 0, 0]); + assert.ok(Number.isNaN(p)); +}); + +test('wilcoxon signed-rank gives small p for clearly nonzero deltas', () => { + const { p } = wilcoxonSignedRank([10, 11, 9, 12, 13, 14, 15, 16, 17, 18]); + assert.ok(p < 0.01, `expected p<0.01, got ${p}`); +}); + +test('analyzeAB on identical A and B reports zero deltas and contains-zero CI', () => { + const fakeRun = ms => ({ run: { proveTotalMs: ms, setupMs: ms * 0.6, proveMs: ms * 0.4, verified: true, proofFieldCount: 2630, verificationKeyBytes: 4576 } }); + const pairs = []; + for (let p = 0; p < 11; p++) { + const ms = 10000 + Math.sin(p) * 100; + pairs.push({ pair: p, variant: 'a', position: p % 2 === 0 ? 0 : 1, warmup: p === 0, ...fakeRun(ms) }); + pairs.push({ pair: p, variant: 'b', position: p % 2 === 0 ? 1 : 0, warmup: p === 0, ...fakeRun(ms) }); + } + const result = { benchmark: 'chonk-ab', flow: 'x', variants: ['a', 'b'], pairs, warmupPairs: 1 }; + const analysis = analyzeAB(result, { bootstrapIters: 1000 }); + assert.equal(analysis.analyzedPairs, 10); + const m = analysis.perMetric.proveTotalMs; + assert.equal(m.deltaMs.median, 0); + assert.ok(m.deltaPct.ci95.lo <= 0 && m.deltaPct.ci95.hi >= 0, `Δ% CI [${m.deltaPct.ci95.lo}, ${m.deltaPct.ci95.hi}] should contain 0`); + assert.equal(m.significant, false); +}); + +test('analyzeAB rejects results without exactly two variants', () => { + assert.throws(() => analyzeAB({ variants: ['a'], pairs: [] }), /2-element array/); +}); + test('server exposes health and pinned input index', async () => { const root = await mkdtemp(join(tmpdir(), 'wasm-bench-static-')); const inputs = await mkdtemp(join(tmpdir(), 'wasm-bench-inputs-')); diff --git a/barretenberg/wasm-bench/src/bench-worker.js b/barretenberg/wasm-bench/src/bench-worker.js index abd5c639a6e9..f6903e155dbc 100644 --- a/barretenberg/wasm-bench/src/bench-worker.js +++ b/barretenberg/wasm-bench/src/bench-worker.js @@ -99,8 +99,14 @@ async function runBench(options) { return smoke; } + progress('crs_fetch_start', { srsSize: options.srsSize, grumpkinSrsSize: options.grumpkinSrsSize }); const crsInfo = await timed(phases, 'fetch_crs_and_init_srs', () => - initSrs(wasm, { srsSize: options.srsSize, grumpkinSrsSize: options.grumpkinSrsSize, crsBaseUrl: options.crsBaseUrl }), + initSrs(wasm, { + srsSize: options.srsSize, + grumpkinSrsSize: options.grumpkinSrsSize, + crsBaseUrl: options.crsBaseUrl, + progress, + }), ); progress('srs_ready', crsInfo); diff --git a/barretenberg/wasm-bench/src/chonk.js b/barretenberg/wasm-bench/src/chonk.js index dc11acdf1cc9..4fac0162ee08 100644 --- a/barretenberg/wasm-bench/src/chonk.js +++ b/barretenberg/wasm-bench/src/chonk.js @@ -35,13 +35,16 @@ export async function fetchAndProcessInputs(inputUrl) { } export async function initSrs(wasm, options) { + const progress = options.progress; const crs = await fetchCrs(options); + progress?.('srs_init_g1_start', { num_points: crs.srsSize, g1Bytes: crs.g1.byteLength, g2Bytes: crs.g2.byteLength }); const srsResponse = bbapiCall( wasm, 'SrsInitSrs', { points_buf: crs.g1, num_points: crs.srsSize, g2_point: crs.g2 }, 'SrsInitSrsResponse', ); + progress?.('srs_init_grumpkin_start', { num_points: crs.grumpkinSrsSize, grumpkinBytes: crs.grumpkin.byteLength }); bbapiCall( wasm, 'SrsInitGrumpkinSrs', diff --git a/barretenberg/wasm-bench/src/crs.js b/barretenberg/wasm-bench/src/crs.js index 24bca6e56695..62baa8160af1 100644 --- a/barretenberg/wasm-bench/src/crs.js +++ b/barretenberg/wasm-bench/src/crs.js @@ -1,23 +1,37 @@ const DEFAULT_SRS_SIZE = 2 ** 19; const DEFAULT_GRUMPKIN_SRS_SIZE = 2 ** 16; -async function fetchBytes(url) { - const response = await fetch(url, { cache: 'force-cache' }); - if (!response.ok) { - throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); +async function fetchBytes(url, progress, label, { retries = 4, retryDelayMs = 500 } = {}) { + const started = performance.now(); + let lastError; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const response = await fetch(url, { cache: 'force-cache' }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + progress?.(label, { url, bytes: bytes.byteLength, elapsedMs: performance.now() - started, attempt }); + return bytes; + } catch (error) { + lastError = error; + progress?.(`${label}_retry`, { url, attempt, message: error?.message ?? String(error) }); + if (attempt >= retries) break; + await new Promise(resolve => setTimeout(resolve, retryDelayMs * Math.pow(2, attempt))); + } } - return new Uint8Array(await response.arrayBuffer()); + throw new Error(`Failed to fetch ${url} after ${retries + 1} attempts: ${lastError?.message ?? lastError}`); } function withBaseUrl(baseUrl, path) { return baseUrl ? new URL(path, baseUrl).toString() : path; } -export async function fetchCrs({ srsSize = DEFAULT_SRS_SIZE, grumpkinSrsSize = DEFAULT_GRUMPKIN_SRS_SIZE, crsBaseUrl } = {}) { +export async function fetchCrs({ srsSize = DEFAULT_SRS_SIZE, grumpkinSrsSize = DEFAULT_GRUMPKIN_SRS_SIZE, crsBaseUrl, progress } = {}) { const [g1, g2, grumpkin] = await Promise.all([ - fetchBytes(withBaseUrl(crsBaseUrl, `/crs/bn254-g1?points=${srsSize}`)), - fetchBytes(withBaseUrl(crsBaseUrl, '/crs/bn254-g2')), - fetchBytes(withBaseUrl(crsBaseUrl, `/crs/grumpkin-g1?points=${grumpkinSrsSize}`)), + fetchBytes(withBaseUrl(crsBaseUrl, `/crs/bn254-g1?points=${srsSize}`), progress, 'crs_g1_fetched'), + fetchBytes(withBaseUrl(crsBaseUrl, '/crs/bn254-g2'), progress, 'crs_g2_fetched'), + fetchBytes(withBaseUrl(crsBaseUrl, `/crs/grumpkin-g1?points=${grumpkinSrsSize}`), progress, 'crs_grumpkin_fetched'), ]); return { srsSize, diff --git a/barretenberg/wasm-bench/src/main.js b/barretenberg/wasm-bench/src/main.js index 1a51d1abe7b2..6efe47adeabd 100644 --- a/barretenberg/wasm-bench/src/main.js +++ b/barretenberg/wasm-bench/src/main.js @@ -3,6 +3,7 @@ const form = document.querySelector('#bench-form'); const runButton = document.querySelector('#run'); function appendLog(value) { + if (!logEl) return; const text = typeof value === 'string' ? value : JSON.stringify(value); logEl.textContent += `${text}\n`; logEl.scrollTop = logEl.scrollHeight; @@ -26,65 +27,233 @@ async function postJson(path, body) { } } -async function run(options) { - runButton.disabled = true; - logEl.textContent = ''; +function setRunButton(disabled) { + if (runButton) runButton.disabled = disabled; +} + +function runWorker(workerOptions, { onProgress } = {}) { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL('./bench-worker.js', import.meta.url), { type: 'module' }); + const terminate = () => worker.terminate(); + worker.onmessage = event => { + const message = event.data; + if (message.type === 'progress') { + onProgress?.(message); + return; + } + if (message.type === 'result') { + terminate(); + resolve(message.result); + return; + } + if (message.type === 'error') { + terminate(); + const err = new Error(message.error?.message || 'worker error'); + err.stack = message.error?.stack || err.stack; + reject(err); + } + }; + worker.onerror = event => { + terminate(); + const err = new Error(event.message || 'worker top-level error'); + reject(err); + }; + worker.postMessage({ type: 'run', options: workerOptions }); + }); +} + +async function runSingle(options) { + setRunButton(true); + if (logEl) logEl.textContent = ''; window.__wasmBenchStatus = { state: 'running', options }; window.__wasmBenchResult = undefined; window.__wasmBenchError = undefined; - const worker = new Worker(new URL('./bench-worker.js', import.meta.url), { type: 'module' }); - worker.onmessage = async event => { - const message = event.data; - if (message.type === 'progress') { - const row = { ...message, options }; - window.__wasmBenchStatus = { state: 'progress', event: message.event, data: message.data }; - appendLog(row); - await postJson('/progress', row); - return; - } + try { + const result = await runWorker(options, { + onProgress: async msg => { + const row = { ...msg, options }; + window.__wasmBenchStatus = { state: 'progress', event: msg.event, data: msg.data }; + appendLog(row); + await postJson('/progress', row); + }, + }); + window.__wasmBenchStatus = { state: 'complete' }; + window.__wasmBenchResult = result; + appendLog(result); + await postJson('/result', result); + } catch (error) { + const errObj = { message: error.message || String(error), stack: error.stack || '' }; + window.__wasmBenchStatus = { state: 'error', error: errObj }; + window.__wasmBenchError = errObj; + appendLog(errObj); + await postJson('/progress', { type: 'error', error: errObj, options }); + } finally { + setRunButton(false); + } +} - if (message.type === 'result') { - window.__wasmBenchStatus = { state: 'complete' }; - window.__wasmBenchResult = message.result; - appendLog(message.result); - await postJson('/result', message.result); - worker.terminate(); - runButton.disabled = false; - return; - } +export function buildABSchedule(variants, pairs) { + if (!Array.isArray(variants) || variants.length !== 2) { + throw new Error(`A/B mode requires exactly 2 variants (got ${JSON.stringify(variants)})`); + } + const schedule = []; + for (let p = 0; p < pairs; p++) { + schedule.push(p % 2 === 0 ? [variants[0], variants[1]] : [variants[1], variants[0]]); + } + return schedule; +} - if (message.type === 'error') { - window.__wasmBenchStatus = { state: 'error', error: message.error }; - window.__wasmBenchError = message.error; - appendLog(message.error); - await postJson('/progress', { type: 'error', error: message.error, options }); - worker.terminate(); - runButton.disabled = false; - } - }; - worker.onerror = async event => { - const error = { message: event.message, stack: event.error?.stack ?? '' }; - window.__wasmBenchStatus = { state: 'error', error }; - window.__wasmBenchError = error; - appendLog(error); - await postJson('/progress', { type: 'error', error, options }); - worker.terminate(); - runButton.disabled = false; +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function runAB(options) { + setRunButton(true); + if (logEl) logEl.textContent = ''; + const variants = options.variants || ['a', 'b']; + const pairs = Math.max(1, Number(options.pairs) || 11); + const warmupPairs = Math.max(0, Number(options.warmupPairs ?? options.warmupRuns ?? 1)); + const wasmBaseUrls = options.wasmBaseUrls || { + [variants[0]]: `/wasm/${variants[0]}`, + [variants[1]]: `/wasm/${variants[1]}`, }; - worker.postMessage({ type: 'run', options }); + const interRunSleepMs = Math.max(0, Number(options.interRunSleepMs) || 0); + + window.__wasmBenchStatus = { state: 'running-ab', options, pairs, variants }; + window.__wasmBenchResult = undefined; + window.__wasmBenchError = undefined; + + const schedule = buildABSchedule(variants, pairs); + const collected = []; + let features; + + appendLog({ event: 'ab_start', variants, pairs, warmupPairs, wasmBaseUrls }); + await postJson('/progress', { type: 'ab_start', variants, pairs, warmupPairs, wasmBaseUrls, options }); + + try { + for (let p = 0; p < pairs; p++) { + const order = schedule[p]; + const pairRuns = []; + for (let position = 0; position < order.length; position++) { + const variant = order[position]; + const wasmBaseUrl = wasmBaseUrls[variant]; + if (!wasmBaseUrl) { + throw new Error(`No wasmBaseUrl configured for variant "${variant}"`); + } + const workerOptions = { + benchmark: 'chonk-prove', + flow: options.flow, + runs: 1, + threads: options.threads, + smoke: false, + wasmBaseUrl, + ...(options.memMaxPages ? { memMaxPages: options.memMaxPages } : {}), + ...(options.srsSize ? { srsSize: options.srsSize } : {}), + ...(options.grumpkinSrsSize ? { grumpkinSrsSize: options.grumpkinSrsSize } : {}), + ...(options.crsBaseUrl ? { crsBaseUrl: options.crsBaseUrl } : {}), + }; + await postJson('/progress', { type: 'pair_start', pair: p, variant, position, warmup: p < warmupPairs }); + appendLog({ event: 'pair_start', pair: p, variant, position }); + const result = await runWorker(workerOptions, { + onProgress: async msg => { + window.__wasmBenchStatus = { state: 'pair_progress', pair: p, variant, event: msg.event }; + await postJson('/progress', { type: 'worker_progress', pair: p, variant, position, inner: msg }); + }, + }); + if (!features) features = result.features; + const run = result.runs && result.runs[0]; + if (!run) { + throw new Error(`Worker produced no run for variant=${variant} pair=${p}`); + } + const pairRun = { + pair: p, + warmup: p < warmupPairs, + variant, + position, + run, + phases: result.phases, + crs: result.crs, + wallMs: result.wallMs, + wasmBaseUrl, + features: result.features, + }; + pairRuns.push(pairRun); + collected.push(pairRun); + await postJson('/progress', { + type: 'pair_run', + pair: p, + variant, + position, + warmup: p < warmupPairs, + proveTotalMs: run.proveTotalMs, + setupMs: run.setupMs, + proveMs: run.proveMs, + verified: run.verified, + proofFieldCount: run.proofFieldCount, + verificationKeyBytes: run.verificationKeyBytes, + }); + appendLog({ + event: 'pair_run', + pair: p, + variant, + position, + proveTotalMs: Math.round(run.proveTotalMs), + setupMs: Math.round(run.setupMs), + proveMs: Math.round(run.proveMs), + }); + if (interRunSleepMs > 0) { + await sleep(interRunSleepMs); + } + } + await postJson('/progress', { type: 'pair_complete', pair: p, runs: pairRuns }); + } + + const result = { + benchmark: 'chonk-ab', + flow: options.flow, + variants, + wasmBaseUrls, + pairs: collected, + pairsCount: pairs, + warmupPairs, + features, + }; + window.__wasmBenchStatus = { state: 'complete' }; + window.__wasmBenchResult = result; + appendLog({ event: 'ab_complete', collected: collected.length }); + await postJson('/result', result); + } catch (error) { + const errObj = { message: error.message || String(error), stack: error.stack || '' }; + window.__wasmBenchStatus = { state: 'error', error: errObj }; + window.__wasmBenchError = errObj; + appendLog({ event: 'ab_error', error: errObj }); + await postJson('/progress', { type: 'ab_error', error: errObj }); + } finally { + setRunButton(false); + } +} + +async function run(options) { + if (options?.benchmark === 'chonk-ab') { + return runAB(options); + } + return runSingle(options); } -form.addEventListener('submit', event => { - event.preventDefault(); - const formData = new FormData(form); - void run({ - flow: String(formData.get('flow')), - threads: String(formData.get('threads')), - runs: Number(formData.get('runs')), - smoke: String(formData.get('smoke')) === 'true', +if (form) { + form.addEventListener('submit', event => { + event.preventDefault(); + const formData = new FormData(form); + void run({ + benchmark: 'chonk-prove', + flow: String(formData.get('flow')), + threads: String(formData.get('threads')), + runs: Number(formData.get('runs')), + smoke: String(formData.get('smoke')) === 'true', + }); }); -}); +} const params = new URLSearchParams(location.search); const benchParam = params.get('bench'); diff --git a/barretenberg/wasm-bench/src/wasm-runtime.js b/barretenberg/wasm-bench/src/wasm-runtime.js index c6fd1c179add..a5605367b41c 100644 --- a/barretenberg/wasm-bench/src/wasm-runtime.js +++ b/barretenberg/wasm-bench/src/wasm-runtime.js @@ -19,7 +19,7 @@ function stringFromMemory(memory, addr) { while (mem[end] !== 0) { end++; } - return new TextDecoder('ascii').decode(mem.subarray(addr >>> 0, end)); + return new TextDecoder('ascii').decode(mem.slice(addr >>> 0, end)); } export function createImportObject({ memory, logger = () => {}, envHardwareConcurrency = () => 1, threadSpawn }) { @@ -55,17 +55,27 @@ export function createImportObject({ memory, logger = () => {}, envHardwareConcu }; } -async function fetchWasmBytes(url, progress) { +async function fetchWasmBytes(url, progress, { retries = 4, retryDelayMs = 500 } = {}) { const started = performance.now(); - const response = await fetch(url, { cache: 'no-store' }); - if (!response.ok) { - throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`); + let lastError; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + const response = await fetch(url, { cache: 'default' }); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + const compressed = new Uint8Array(await response.arrayBuffer()); + progress?.('wasm_fetch', { url, bytes: compressed.byteLength, elapsedMs: performance.now() - started, attempt }); + const isGzip = compressed[0] === 0x1f && compressed[1] === 0x8b && compressed[2] === 0x08; + return isGzip ? pako.ungzip(compressed) : compressed; + } catch (error) { + lastError = error; + progress?.('wasm_fetch_retry', { url, attempt, message: error?.message ?? String(error) }); + if (attempt >= retries) break; + await new Promise(resolve => setTimeout(resolve, retryDelayMs * Math.pow(2, attempt))); + } } - const compressed = new Uint8Array(await response.arrayBuffer()); - progress?.('wasm_fetch', { url, bytes: compressed.byteLength, elapsedMs: performance.now() - started }); - - const isGzip = compressed[0] === 0x1f && compressed[1] === 0x8b && compressed[2] === 0x08; - return isGzip ? pako.ungzip(compressed) : compressed; + throw new Error(`Failed to fetch ${url} after ${retries + 1} attempts: ${lastError?.message ?? lastError}`); } function copyOut(memory, start, end) {