diff --git a/.claude/skills/wasm-bench-browserstack/SKILL.md b/.claude/skills/wasm-bench-browserstack/SKILL.md new file mode 100644 index 000000000000..e52fa95fd8ff --- /dev/null +++ b/.claude/skills/wasm-bench-browserstack/SKILL.md @@ -0,0 +1,121 @@ +--- +name: wasm-bench-browserstack +description: Run on-demand bb.js Chonk prove+verify benchmarks on BrowserStack real mobile devices using pinned IVC inputs. +--- + +# BrowserStack bb.js Chonk Benchmark + +Use this skill when asked to benchmark bb.js Chonk proving on real browsers or +mobile devices. This is an on-demand benchmark, not a per-PR CI gate. + +## Scope + +- Benchmark full Chonk prove plus verify through `window.proveChonk` and + `window.verifyChonk`. +- Use the pinned Chonk IVC input archive from + `barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh`. +- Default device coverage is old iOS, new iOS, old Android, and new Android. + Override the matrix for focused device work. +- Results are JSONL rows, one row per input per device, including BrowserStack + requested capabilities and browser feature probes for SAB, threads, SIMD, and + hardware concurrency. + +## Prerequisites + +From the repository root: + +```bash +cd barretenberg/cpp +./scripts/test_chonk_standalone_vks_havent_changed.sh --download_pinned_inputs + +cd ../ts +yarn build:wasm +yarn build:browser + +cd ../acir_tests +yarn install +yarn workspace browser-test-app build +``` + +Set BrowserStack credentials: + +```bash +export BROWSERSTACK_USER_NAME=... +export BROWSERSTACK_ACCESS_KEY=... +``` + +`BROWSERSTACK_USERNAME` is also accepted for the username, and +`BROWSERSTACK_KEY` is also accepted for the access key. + +## Run + +```bash +cd barretenberg/acir_tests +yarn workspace headless-test browserstack:chonk-bench \ + --output browserstack-chonk-bench.jsonl +``` + +The default flow set is: + +- `ecdsar1+transfer_0_recursions+sponsored_fpc` +- `ecdsar1+transfer_1_recursions+sponsored_fpc` +- `ecdsar1+token_bridge_claim_private+sponsored_fpc` + +Override flows: + +```bash +yarn workspace headless-test browserstack:chonk-bench \ + --flow ecdsar1+transfer_1_recursions+sponsored_fpc \ + --flow ecdsar1+transfer_0_recursions+sponsored_fpc +``` + +Use explicit input files: + +```bash +yarn workspace headless-test browserstack:chonk-bench \ + --input /path/to/ivc-inputs-a.msgpack \ + --input /path/to/ivc-inputs-b.msgpack +``` + +Override the device matrix with JSON or a JSON file: + +```json +[ + { + "name": "ios-17", + "capabilities": { + "browserName": "safari", + "bstack:options": { + "deviceName": "iPhone 15", + "osVersion": "17", + "realMobile": "true" + } + } + } +] +``` + +Then: + +```bash +yarn workspace headless-test browserstack:chonk-bench --matrix ./matrix.json +``` + +## Output + +Each JSONL row contains: + +- `device` +- `requested_capabilities` +- `browser_features` +- `input` +- `input_bytes` +- `proof_bytes` +- `verification_key_bytes` +- `prove_ms` +- `verify_ms` +- `total_ms` +- `verified` + +Keep the raw JSONL with the PR or gist so later benchmark comparisons can reuse +the same flow/device inputs. diff --git a/barretenberg/acir_tests/README.md b/barretenberg/acir_tests/README.md index e3c232874bd9..20bd3d95c4e7 100644 --- a/barretenberg/acir_tests/README.md +++ b/barretenberg/acir_tests/README.md @@ -27,6 +27,7 @@ The easiest way to find how to run specific test(s): ``` This will show you the exact commands used in CI. For example: + ``` c5f89...:ISOLATE=1 scripts/bb_prove_sol_verify.sh assert_statement --disable_zk c5f89...:ISOLATE=1 scripts/bb_prove_sol_verify.sh assert_statement @@ -35,6 +36,7 @@ c5f89... scripts/bb_prove.sh assert_statement ``` You can run any of these commands directly (ignore the hash prefix): + ```bash scripts/bb_prove.sh assert_statement ``` @@ -44,3 +46,32 @@ Programmatically, you can also do from root: ```bash ./barretenberg/acir_tests/bootstrap.sh test_cmds | grep assert_statement | ci3/parallelize ``` + +## BrowserStack Chonk Benchmark + +The headless browser harness can run on-demand bb.js Chonk prove+verify +benchmarks on BrowserStack real devices. It serves the local +`browser-test-app/dest` bundle through BrowserStack Local, so the benchmark uses +the exact local bb.js build and pinned Chonk inputs without uploading artifacts. + +```bash +cd barretenberg/cpp +./scripts/test_chonk_standalone_vks_havent_changed.sh --download_pinned_inputs + +cd ../ts +yarn build:wasm +yarn build:browser + +cd ../acir_tests +yarn workspace browser-test-app build + +export BROWSERSTACK_USER_NAME=... +export BROWSERSTACK_ACCESS_KEY=... +yarn workspace headless-test browserstack:chonk-bench \ + --output browserstack-chonk-bench.jsonl +``` + +The default matrix covers old and new iOS plus old and new Android. Use +`--matrix ./matrix.json` to pass BrowserStack capabilities for a custom device +set, `--flow ` to pick pinned flow folders, or `--input ` for +explicit `ivc-inputs.msgpack` files. diff --git a/barretenberg/acir_tests/browser-test-app/src/index.ts b/barretenberg/acir_tests/browser-test-app/src/index.ts index af4bf7202a53..b3d94a507fb3 100644 --- a/barretenberg/acir_tests/browser-test-app/src/index.ts +++ b/barretenberg/acir_tests/browser-test-app/src/index.ts @@ -105,7 +105,24 @@ function installChonkGlobal() { return { proof, verificationKey }; } + async function verifyChonk( + proof: Uint8Array, + verificationKey: Uint8Array, + threads = 1 + ): Promise { + const { AztecClientBackend } = await import("@aztec/bb.js"); + + const bb = await Barretenberg.new({ threads, logger: bbLogger }); + const backend = new AztecClientBackend([], bb, []); + try { + return await backend.verify(proof, verificationKey); + } finally { + await bb.destroy(); + } + } + (window as any).proveChonk = proveChonk; + (window as any).verifyChonk = verifyChonk; } installChonkGlobal(); diff --git a/barretenberg/acir_tests/headless-test/package.json b/barretenberg/acir_tests/headless-test/package.json index 1f16ffb506ed..88ea3ef92e9f 100644 --- a/barretenberg/acir_tests/headless-test/package.json +++ b/barretenberg/acir_tests/headless-test/package.json @@ -5,15 +5,21 @@ "license": "MIT", "type": "module", "scripts": { - "start": "ts-node-esm ./src/index.ts" + "start": "ts-node-esm ./src/index.ts", + "browserstack:chonk-bench": "node --loader ts-node/esm ./src/browserstack-bench.ts", + "test:browserstack-bench": "node --loader ts-node/esm --test ./src/browserstack-bench.test.ts" }, "dependencies": { + "browserstack-local": "^1.5.13", "chalk": "^5.3.0", "commander": "^12.1.0", "playwright": "1.49.0", - "puppeteer": "^24.22.3" + "puppeteer": "^24.22.3", + "selenium-webdriver": "^4.43.0" }, "devDependencies": { + "@types/node": "20", + "@types/selenium-webdriver": "^4.35.5", "ts-node": "^10.9.2", "typescript": "^5.4.2" }, diff --git a/barretenberg/acir_tests/headless-test/src/browserstack-bench.test.ts b/barretenberg/acir_tests/headless-test/src/browserstack-bench.test.ts new file mode 100644 index 000000000000..1e0756de65e4 --- /dev/null +++ b/barretenberg/acir_tests/headless-test/src/browserstack-bench.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + DEFAULT_DEVICE_MATRIX, + parseDeviceMatrix, + requireBrowserStackUsername, + resolveInputFiles, + startBenchServer, +} from './browserstack-bench.js'; + +const customMatrix = parseDeviceMatrix( + JSON.stringify([ + { + name: 'old-ios', + capabilities: { + browserName: 'safari', + 'bstack:options': { + deviceName: 'iPhone 11', + osVersion: '15', + realMobile: 'true', + }, + }, + }, + ]), +); + +assert.equal(customMatrix.length, 1); +assert.equal(customMatrix[0].name, 'old-ios'); +assert.equal(customMatrix[0].capabilities.browserName, 'safari'); +assert.equal(customMatrix[0].capabilities['bstack:options'].deviceName, 'iPhone 11'); + +const resolvedFlows = resolveInputFiles( + '/repo/yarn-project/end-to-end/example-app-ivc-inputs-out', + ['flow-a', 'flow-b'], + [], +); +assert.deepEqual(resolvedFlows, [ + { + label: 'flow-a', + path: '/repo/yarn-project/end-to-end/example-app-ivc-inputs-out/flow-a/ivc-inputs.msgpack', + route: '/inputs/0-flow-a.msgpack', + }, + { + label: 'flow-b', + path: '/repo/yarn-project/end-to-end/example-app-ivc-inputs-out/flow-b/ivc-inputs.msgpack', + route: '/inputs/1-flow-b.msgpack', + }, +]); + +const resolvedExplicitInputs = resolveInputFiles('/ignored', [], ['/tmp/a.msgpack', '/tmp/b.msgpack']); +assert.deepEqual( + resolvedExplicitInputs.map(input => ({ label: input.label, route: input.route })), + [ + { label: 'a', route: '/inputs/0-a.msgpack' }, + { label: 'b', route: '/inputs/1-b.msgpack' }, + ], +); + +assert.equal(DEFAULT_DEVICE_MATRIX.length, 4); +assert.deepEqual( + DEFAULT_DEVICE_MATRIX.map(device => device.name), + ['old-ios', 'new-ios', 'old-android', 'new-android'], +); + +assert.equal(requireBrowserStackUsername({ BROWSERSTACK_USER_NAME: 'preferred-name' }), 'preferred-name'); +assert.equal(requireBrowserStackUsername({ BROWSERSTACK_USERNAME: 'legacy-name' }), 'legacy-name'); +assert.throws(() => requireBrowserStackUsername({}), /BROWSERSTACK_USER_NAME or BROWSERSTACK_USERNAME/); + +const tempRoot = mkdtempSync(join(tmpdir(), 'browserstack-bench-test-')); +const distPath = join(tempRoot, 'dist'); +const inputPath = join(tempRoot, 'ivc-inputs.msgpack'); +mkdirSync(distPath, { recursive: true }); +writeFileSync(join(distPath, 'index.html'), 'ok'); +writeFileSync(inputPath, Buffer.from([1, 2, 3, 4])); + +const { port, server } = await startBenchServer(distPath, [ + { label: 'flow-a', path: inputPath, route: '/inputs/0-flow-a.msgpack' }, +]); +try { + const indexResponse = await fetch(`http://127.0.0.1:${port}/index.html`); + assert.equal(indexResponse.status, 200); + assert.equal(indexResponse.headers.get('cross-origin-opener-policy'), 'same-origin'); + assert.equal(indexResponse.headers.get('cross-origin-embedder-policy'), 'require-corp'); + assert.equal(await indexResponse.text(), 'ok'); + + const inputResponse = await fetch(`http://127.0.0.1:${port}/inputs/0-flow-a.msgpack`); + assert.equal(inputResponse.status, 200); + assert.deepEqual([...new Uint8Array(await inputResponse.arrayBuffer())], [1, 2, 3, 4]); + + const traversalResponse = await fetch(`http://127.0.0.1:${port}/../package.json`); + assert.equal(traversalResponse.status, 404); +} finally { + server.close(); +} diff --git a/barretenberg/acir_tests/headless-test/src/browserstack-bench.ts b/barretenberg/acir_tests/headless-test/src/browserstack-bench.ts new file mode 100644 index 000000000000..f196a6db543b --- /dev/null +++ b/barretenberg/acir_tests/headless-test/src/browserstack-bench.ts @@ -0,0 +1,564 @@ +import fs from 'fs'; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'http'; +import { createWriteStream } from 'fs'; +import { basename, dirname, extname, join, resolve } from 'path'; +import { pathToFileURL, fileURLToPath } from 'url'; + +import browserstack from 'browserstack-local'; +import { Command } from 'commander'; +import { Builder, type WebDriver } from 'selenium-webdriver'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const repoRoot = resolve(__dirname, '..', '..', '..', '..'); + +export type DeviceConfig = { + name: string; + capabilities: Record; +}; + +export type BenchInput = { + label: string; + path: string; + route: string; +}; + +type BrowserFeatureReport = { + userAgent: string; + hardwareConcurrency: number | null; + crossOriginIsolated: boolean; + sharedArrayBuffer: boolean; + wasmSimd: boolean; + wasmThreads: boolean; +}; + +type BrowserBenchRun = { + label: string; + inputBytes: number; + proofBytes: number; + verificationKeyBytes: number; + proveMs: number; + verifyMs: number; + totalMs: number; + verified: boolean; +}; + +type BrowserBenchResult = { + features: BrowserFeatureReport; + runs: BrowserBenchRun[]; +}; + +const DEFAULT_INPUTS_ROOT = join(repoRoot, 'yarn-project', 'end-to-end', 'example-app-ivc-inputs-out'); +const DEFAULT_BROWSER_DIST = join(__dirname, '..', '..', 'browser-test-app', 'dest'); + +const DEFAULT_FLOWS = [ + 'ecdsar1+transfer_0_recursions+sponsored_fpc', + 'ecdsar1+transfer_1_recursions+sponsored_fpc', + 'ecdsar1+token_bridge_claim_private+sponsored_fpc', +]; + +export const DEFAULT_DEVICE_MATRIX: DeviceConfig[] = [ + { + name: 'old-ios', + capabilities: { + browserName: 'safari', + 'bstack:options': { + deviceName: 'iPhone 11', + osVersion: '15', + realMobile: 'true', + }, + }, + }, + { + name: 'new-ios', + capabilities: { + browserName: 'safari', + 'bstack:options': { + deviceName: 'iPhone 15', + osVersion: '17', + realMobile: 'true', + }, + }, + }, + { + name: 'old-android', + capabilities: { + browserName: 'chrome', + 'bstack:options': { + deviceName: 'Samsung Galaxy S10', + osVersion: '9.0', + realMobile: 'true', + }, + }, + }, + { + name: 'new-android', + capabilities: { + browserName: 'chrome', + 'bstack:options': { + deviceName: 'Samsung Galaxy S24', + osVersion: '14.0', + realMobile: 'true', + }, + }, + }, +]; + +export function parseDeviceMatrix(rawMatrix: string): DeviceConfig[] { + const parsed = JSON.parse(rawMatrix); + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error('BrowserStack matrix must be a non-empty JSON array.'); + } + + return parsed.map((device, index) => { + if (!device || typeof device !== 'object') { + throw new Error(`BrowserStack matrix entry ${index} must be an object.`); + } + if (typeof device.name !== 'string' || device.name.length === 0) { + throw new Error(`BrowserStack matrix entry ${index} needs a non-empty name.`); + } + if (!device.capabilities || typeof device.capabilities !== 'object') { + throw new Error(`BrowserStack matrix entry ${device.name} needs capabilities.`); + } + return { + name: device.name, + capabilities: device.capabilities, + }; + }); +} + +export function resolveInputFiles(inputsRoot: string, flows: string[], explicitInputs: string[]): BenchInput[] { + const chosenInputs = + explicitInputs.length > 0 + ? explicitInputs.map(inputPath => { + const label = basename(inputPath).replace(/\.msgpack$/, ''); + return { label, path: resolve(inputPath) }; + }) + : flows.map(flow => ({ + label: flow, + path: join(inputsRoot, flow, 'ivc-inputs.msgpack'), + })); + + return chosenInputs.map((input, index) => ({ + label: input.label, + path: input.path, + route: `/inputs/${index}-${safeRouteLabel(input.label)}.msgpack`, + })); +} + +function safeRouteLabel(label: string): string { + return label.replace(/[^A-Za-z0-9_.-]/g, '_'); +} + +function readMatrixOption(matrixOption?: string): DeviceConfig[] { + if (!matrixOption) { + return DEFAULT_DEVICE_MATRIX; + } + const matrixText = fs.existsSync(matrixOption) ? fs.readFileSync(matrixOption, 'utf8') : matrixOption; + return parseDeviceMatrix(matrixText); +} + +function collectOption(value: string, previous: string[]): string[] { + previous.push(value); + return previous; +} + +function contentType(filePath: string): string { + switch (extname(filePath)) { + case '.js': + return 'application/javascript; charset=utf-8'; + case '.wasm': + return 'application/wasm'; + case '.json': + return 'application/json'; + case '.css': + return 'text/css'; + case '.html': + return 'text/html; charset=utf-8'; + case '.msgpack': + return 'application/octet-stream'; + default: + return 'application/octet-stream'; + } +} + +function addIsolationHeaders(res: ServerResponse) { + res.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp'); + res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); + res.setHeader('Cache-Control', 'no-store'); +} + +function serveFile(res: ServerResponse, filePath: string) { + const content = fs.readFileSync(filePath); + res.writeHead(200, { + 'Content-Type': contentType(filePath), + 'Content-Length': content.length, + }); + res.end(content); +} + +function resolveStaticPath(distPath: string, requestPath: string): string | undefined { + const candidate = resolve(distPath, requestPath.replace(/^\/+/, '')); + const distRoot = resolve(distPath); + if (candidate !== distRoot && !candidate.startsWith(`${distRoot}/`)) { + return undefined; + } + return candidate; +} + +export function startBenchServer(distPath: string, inputs: BenchInput[]): Promise<{ port: number; server: Server }> { + if (!fs.existsSync(distPath)) { + throw new Error(`Browser test app dist not found at ${distPath}. Build browser-test-app first.`); + } + for (const input of inputs) { + if (!fs.existsSync(input.path)) { + throw new Error(`Pinned Chonk input not found: ${input.path}`); + } + } + + const inputRoutes = new Map(inputs.map(input => [input.route, input.path])); + const server = createServer((req: IncomingMessage, res: ServerResponse) => { + addIsolationHeaders(res); + const requestPath = new URL(req.url || '/', 'http://localhost').pathname; + + try { + if (requestPath === '/') { + res.writeHead(302, { Location: '/index.html' }); + res.end(); + return; + } + + const inputPath = inputRoutes.get(requestPath); + if (inputPath) { + serveFile(res, inputPath); + return; + } + + const staticPath = resolveStaticPath(distPath, requestPath); + if (staticPath && fs.existsSync(staticPath) && fs.statSync(staticPath).isFile()) { + serveFile(res, staticPath); + return; + } + + res.writeHead(404); + res.end(`Not found: ${requestPath}`); + } catch (error: any) { + res.writeHead(500); + res.end(`Server error: ${error.message}`); + } + }); + + return new Promise((resolvePromise, reject) => { + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (!address || typeof address !== 'object') { + reject(new Error('Could not bind BrowserStack benchmark server.')); + return; + } + resolvePromise({ port: address.port, server }); + }); + server.on('error', reject); + }); +} + +async function withBrowserStackLocal(key: string, localIdentifier: string, fn: () => Promise): Promise { + const local = new browserstack.Local(); + await new Promise((resolvePromise, reject) => { + local.start( + { + key, + localIdentifier, + forceLocal: true, + onlyAutomate: true, + }, + error => { + if (error) { + reject(error); + return; + } + resolvePromise(); + }, + ); + }); + + try { + return await fn(); + } finally { + await new Promise(resolvePromise => { + local.stop(() => { + resolvePromise(); + }); + }); + } +} + +function mergedCapabilities( + device: DeviceConfig, + localIdentifier: string, + projectName: string, + buildName: string, +): Record { + const capabilities = structuredClone(device.capabilities); + const bstackOptions = { + ...(capabilities['bstack:options'] ?? {}), + local: 'true', + localIdentifier, + projectName, + buildName, + sessionName: `bb.js Chonk BrowserStack bench: ${device.name}`, + }; + capabilities['bstack:options'] = bstackOptions; + return capabilities; +} + +async function buildDriver(username: string, accessKey: string, capabilities: Record): Promise { + return new Builder() + .usingServer(`https://${username}:${accessKey}@hub-cloud.browserstack.com/wd/hub`) + .withCapabilities(capabilities) + .build(); +} + +async function setBrowserStackStatus(driver: WebDriver, passed: boolean, reason: string) { + const payload = JSON.stringify({ + action: 'setSessionStatus', + arguments: { + status: passed ? 'passed' : 'failed', + reason, + }, + }); + try { + await driver.executeScript(`browserstack_executor: ${payload}`); + } catch { + // Session status is best-effort and should not mask benchmark results. + } +} + +async function runBrowserBench( + driver: WebDriver, + url: string, + inputs: BenchInput[], + threads: number, + timeoutMs: number, +): Promise { + await driver.manage().setTimeouts({ pageLoad: 90_000, script: timeoutMs }); + await driver.get(url); + + await driver.wait(async () => { + return await driver.executeScript( + 'return typeof window.proveChonk === "function" && typeof window.verifyChonk === "function";', + ); + }, 90_000); + + const result = (await driver.executeAsyncScript( + ` +const inputs = arguments[0]; +const threads = arguments[1]; +const done = arguments[arguments.length - 1]; + +function validateWasm(bytes) { + try { + return WebAssembly.validate(new Uint8Array(bytes)); + } catch (_) { + return false; + } +} + +(async () => { + const features = { + userAgent: navigator.userAgent, + hardwareConcurrency: navigator.hardwareConcurrency || null, + crossOriginIsolated: Boolean(self.crossOriginIsolated), + sharedArrayBuffer: typeof SharedArrayBuffer !== 'undefined', + wasmSimd: validateWasm([0,97,115,109,1,0,0,0,1,5,1,96,0,1,123,3,2,1,0,10,10,1,8,0,65,0,253,12,26,11]), + wasmThreads: typeof SharedArrayBuffer !== 'undefined' && typeof Atomics !== 'undefined', + }; + const runs = []; + for (const input of inputs) { + const response = await fetch(input.route, { cache: 'no-store' }); + if (!response.ok) { + throw new Error('Failed to fetch ' + input.route + ': ' + response.status); + } + const ivcInputs = new Uint8Array(await response.arrayBuffer()); + const proveStart = performance.now(); + const { proof, verificationKey } = await window.proveChonk(ivcInputs, threads); + const proveMs = performance.now() - proveStart; + const verifyStart = performance.now(); + const verified = await window.verifyChonk(proof, verificationKey, 1); + const verifyMs = performance.now() - verifyStart; + runs.push({ + label: input.label, + inputBytes: ivcInputs.byteLength, + proofBytes: proof.byteLength, + verificationKeyBytes: verificationKey.byteLength, + proveMs, + verifyMs, + totalMs: proveMs + verifyMs, + verified, + }); + if (!verified) { + throw new Error('Chonk proof did not verify for ' + input.label); + } + } + done({ ok: true, value: { features, runs } }); +})().catch(error => done({ + ok: false, + error: { + message: error && error.message ? error.message : String(error), + stack: error && error.stack ? error.stack : '', + }, +})); +`, + inputs.map(input => ({ label: input.label, route: input.route })), + threads, + )) as { ok: true; value: BrowserBenchResult } | { ok: false; error: { message: string; stack: string } }; + + if (!result.ok) { + throw new Error(`${result.error.message}\n${result.error.stack}`); + } + return result.value; +} + +export function requireBrowserStackUsername(env: NodeJS.ProcessEnv = process.env): string { + const value = env.BROWSERSTACK_USER_NAME || env.BROWSERSTACK_USERNAME; + if (!value) { + throw new Error('BROWSERSTACK_USER_NAME or BROWSERSTACK_USERNAME is required.'); + } + return value; +} + +function writeDeviceRows( + output: fs.WriteStream, + device: DeviceConfig, + capabilities: Record, + result: BrowserBenchResult, +) { + const recordedAt = new Date().toISOString(); + for (const run of result.runs) { + output.write( + `${JSON.stringify({ + recorded_at: recordedAt, + device: device.name, + requested_capabilities: capabilities, + browser_features: result.features, + input: run.label, + input_bytes: run.inputBytes, + proof_bytes: run.proofBytes, + verification_key_bytes: run.verificationKeyBytes, + prove_ms: run.proveMs, + verify_ms: run.verifyMs, + total_ms: run.totalMs, + verified: run.verified, + })}\n`, + ); + } +} + +async function runCli() { + const program = new Command('browserstack_chonk_bench'); + program + .description('Run bb.js Chonk prove+verify benchmarks on BrowserStack real devices.') + .option('--inputs-root ', 'Directory containing pinned Chonk flow folders.', DEFAULT_INPUTS_ROOT) + .option('--flow ', 'Pinned Chonk flow folder to run. Repeatable.', collectOption, []) + .option('--input ', 'Explicit ivc-inputs.msgpack path. Repeatable; overrides --flow.', collectOption, []) + .option('--matrix ', 'BrowserStack device matrix as JSON or a path to JSON.') + .option('--browser-dist ', 'Built browser-test-app/dest directory.', DEFAULT_BROWSER_DIST) + .option('--output ', 'JSONL output path.', 'browserstack-chonk-bench.jsonl') + .option('--threads ', 'Requested bb.js worker count.', value => Number.parseInt(value, 10), 4) + .option( + '--timeout-ms ', + 'Per-device script timeout in milliseconds.', + value => Number.parseInt(value, 10), + 900_000, + ) + .option('--project-name ', 'BrowserStack project name.', 'barretenberg bb.js') + .option('--build-name ', 'BrowserStack build name.', `chonk-browser-bench-${new Date().toISOString()}`) + .option('--local-identifier ', 'BrowserStack Local identifier.', `bb-browser-bench-${process.pid}`); + + const options = program.parse().opts<{ + inputsRoot: string; + flow: string[]; + input: string[]; + matrix?: string; + browserDist: string; + output: string; + threads: number; + timeoutMs: number; + projectName: string; + buildName: string; + localIdentifier: string; + }>(); + + const username = requireBrowserStackUsername(); + const accessKey = process.env.BROWSERSTACK_ACCESS_KEY || process.env.BROWSERSTACK_KEY; + if (!accessKey) { + throw new Error('BROWSERSTACK_ACCESS_KEY or BROWSERSTACK_KEY is required.'); + } + + const matrix = readMatrixOption(options.matrix); + const inputs = resolveInputFiles( + resolve(options.inputsRoot), + options.flow.length > 0 ? options.flow : DEFAULT_FLOWS, + options.input, + ); + const { port, server } = await startBenchServer(resolve(options.browserDist), inputs); + const output = createWriteStream(options.output, { flags: 'a' }); + const browserUrl = `http://bs-local.com:${port}/index.html`; + + console.log(`Serving browser-test-app to BrowserStack at ${browserUrl}`); + console.log(`Writing JSONL results to ${options.output}`); + + try { + await withBrowserStackLocal(accessKey, options.localIdentifier, async () => { + const failures: string[] = []; + for (const device of matrix) { + const capabilities = mergedCapabilities( + device, + options.localIdentifier, + options.projectName, + options.buildName, + ); + console.log(`Running ${device.name}...`); + let driver: WebDriver | undefined; + try { + driver = await buildDriver(username, accessKey, capabilities); + const result = await runBrowserBench(driver, browserUrl, inputs, options.threads, options.timeoutMs); + writeDeviceRows(output, device, capabilities, result); + await setBrowserStackStatus(driver, true, `${result.runs.length} Chonk input(s) proven and verified.`); + console.log(`Finished ${device.name}.`); + } catch (error: any) { + output.write( + `${JSON.stringify({ + recorded_at: new Date().toISOString(), + device: device.name, + requested_capabilities: capabilities, + error: error.message, + ok: false, + })}\n`, + ); + if (driver) { + await setBrowserStackStatus(driver, false, error.message); + } + failures.push(`${device.name}: ${error.message}`); + } finally { + if (driver) { + await driver.quit(); + } + } + } + if (failures.length > 0) { + throw new Error(`BrowserStack benchmark failed on ${failures.length} device(s):\n${failures.join('\n')}`); + } + }); + } finally { + output.end(); + server.close(); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runCli().catch(error => { + console.error(error); + process.exit(1); + }); +} diff --git a/barretenberg/acir_tests/yarn.lock b/barretenberg/acir_tests/yarn.lock index 761c7b6cbd9d..14df8f87696d 100644 --- a/barretenberg/acir_tests/yarn.lock +++ b/barretenberg/acir_tests/yarn.lock @@ -68,6 +68,13 @@ __metadata: languageName: node linkType: hard +"@bazel/runfiles@npm:^6.5.0": + version: 6.5.0 + resolution: "@bazel/runfiles@npm:6.5.0" + checksum: 10c0/4e60ff52c8503bf47e142d7835daf08d7cf6ee0047336954c5fb0d4b7d87ce00494d573eeb14571d51232a5da215258fa052278e4b0c9f6e3a65aca494cad945 + languageName: node + linkType: hard + "@cspotcode/source-map-support@npm:^0.8.0": version: 0.8.1 resolution: "@cspotcode/source-map-support@npm:0.8.1" @@ -524,6 +531,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:20": + version: 20.19.40 + resolution: "@types/node@npm:20.19.40" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/50b51ec30e213d6255032fc0672799f879f1f031ed0e7ddde865ee4482a57e64a2bc0e4cd8b52fdbc92d403ce8475f8eaca76e4b08b95c11fd1310a702b3c652 + languageName: node + linkType: hard + "@types/node@npm:22.7.5": version: 22.7.5 resolution: "@types/node@npm:22.7.5" @@ -561,6 +577,16 @@ __metadata: languageName: node linkType: hard +"@types/selenium-webdriver@npm:^4.35.5": + version: 4.35.5 + resolution: "@types/selenium-webdriver@npm:4.35.5" + dependencies: + "@types/node": "npm:*" + "@types/ws": "npm:*" + checksum: 10c0/ddceff2632ac4908ad90db1c4cbd88999ced764f9f37dc09a3ef6a4d904b13b8164a695cb29bef2df5a45ab5ed289b8121b3c91b13936d589aea6f61b8ed9100 + languageName: node + linkType: hard + "@types/send@npm:*": version: 0.17.4 resolution: "@types/send@npm:0.17.4" @@ -600,6 +626,15 @@ __metadata: languageName: node linkType: hard +"@types/ws@npm:*": + version: 8.18.1 + resolution: "@types/ws@npm:8.18.1" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/61aff1129143fcc4312f083bc9e9e168aa3026b7dd6e70796276dcfb2c8211c4292603f9c4864fae702f2ed86e4abd4d38aa421831c2fd7f856c931a481afbab + languageName: node + linkType: hard + "@types/ws@npm:^8.5.10": version: 8.5.14 resolution: "@types/ws@npm:8.5.14" @@ -946,6 +981,15 @@ __metadata: languageName: node linkType: hard +"agent-base@npm:6, agent-base@npm:^6.0.2": + version: 6.0.2 + resolution: "agent-base@npm:6.0.2" + dependencies: + debug: "npm:4" + checksum: 10c0/dc4f757e40b5f3e3d674bc9beb4f1048f4ee83af189bae39be99f57bf1f48dde166a8b0a5342a84b5944ee8e6ed1e5a9d801858f4ad44764e84957122fe46261 + languageName: node + linkType: hard + "agent-base@npm:^7.1.0, agent-base@npm:^7.1.2": version: 7.1.3 resolution: "agent-base@npm:7.1.3" @@ -1337,6 +1381,18 @@ __metadata: languageName: node linkType: hard +"browserstack-local@npm:^1.5.13": + version: 1.5.13 + resolution: "browserstack-local@npm:1.5.13" + dependencies: + agent-base: "npm:^6.0.2" + https-proxy-agent: "npm:^5.0.1" + is-running: "npm:^2.1.0" + tree-kill: "npm:^1.2.2" + checksum: 10c0/41493365cfa6c89625caef732d444827818c8b4f1664d0bc2c76a56151332cfd85bced15f57bf6fcec8d85671c62eeeeb864a147d06b32fe8cfcc0279d7dba4e + languageName: node + linkType: hard + "buffer-crc32@npm:~0.2.3": version: 0.2.13 resolution: "buffer-crc32@npm:0.2.13" @@ -2663,10 +2719,14 @@ __metadata: version: 0.0.0-use.local resolution: "headless-test@workspace:headless-test" dependencies: + "@types/node": "npm:20" + "@types/selenium-webdriver": "npm:^4.35.5" + browserstack-local: "npm:^1.5.13" chalk: "npm:^5.3.0" commander: "npm:^12.1.0" playwright: "npm:1.49.0" puppeteer: "npm:^24.22.3" + selenium-webdriver: "npm:^4.43.0" ts-node: "npm:^10.9.2" typescript: "npm:^5.4.2" languageName: unknown @@ -2819,6 +2879,16 @@ __metadata: languageName: node linkType: hard +"https-proxy-agent@npm:^5.0.1": + version: 5.0.1 + resolution: "https-proxy-agent@npm:5.0.1" + dependencies: + agent-base: "npm:6" + debug: "npm:4" + checksum: 10c0/6dd639f03434003577c62b27cafdb864784ef19b2de430d8ae2a1d45e31c4fd60719e5637b44db1a88a046934307da7089e03d6089ec3ddacc1189d8de8897d1 + languageName: node + linkType: hard + "https-proxy-agent@npm:^7.0.1, https-proxy-agent@npm:^7.0.6": version: 7.0.6 resolution: "https-proxy-agent@npm:7.0.6" @@ -2868,6 +2938,13 @@ __metadata: languageName: node linkType: hard +"immediate@npm:~3.0.5": + version: 3.0.6 + resolution: "immediate@npm:3.0.6" + checksum: 10c0/f8ba7ede69bee9260241ad078d2d535848745ff5f6995c7c7cb41cfdc9ccc213f66e10fa5afb881f90298b24a3f7344b637b592beb4f54e582770cdce3f1f039 + languageName: node + linkType: hard + "import-fresh@npm:^3.3.0": version: 3.3.1 resolution: "import-fresh@npm:3.3.1" @@ -3063,6 +3140,13 @@ __metadata: languageName: node linkType: hard +"is-running@npm:^2.1.0": + version: 2.1.0 + resolution: "is-running@npm:2.1.0" + checksum: 10c0/3caf610508336e7b4d3f63323138ea479f38b7f74c9318254c6c999e06cc9f9bc23727183bb103a3564d517c0b3b9ac057768b3a681c6a63a246089c3ac01c8b + languageName: node + linkType: hard + "is-stream@npm:^2.0.0": version: 2.0.1 resolution: "is-stream@npm:2.0.1" @@ -3186,6 +3270,18 @@ __metadata: languageName: node linkType: hard +"jszip@npm:^3.10.1": + version: 3.10.1 + resolution: "jszip@npm:3.10.1" + dependencies: + lie: "npm:~3.3.0" + pako: "npm:~1.0.2" + readable-stream: "npm:~2.3.6" + setimmediate: "npm:^1.0.5" + checksum: 10c0/58e01ec9c4960383fb8b38dd5f67b83ccc1ec215bf74c8a5b32f42b6e5fb79fada5176842a11409c4051b5b94275044851814a31076bf49e1be218d3ef57c863 + languageName: node + linkType: hard + "kind-of@npm:^6.0.2": version: 6.0.3 resolution: "kind-of@npm:6.0.3" @@ -3203,6 +3299,15 @@ __metadata: languageName: node linkType: hard +"lie@npm:~3.3.0": + version: 3.3.0 + resolution: "lie@npm:3.3.0" + dependencies: + immediate: "npm:~3.0.5" + checksum: 10c0/56dd113091978f82f9dc5081769c6f3b947852ecf9feccaf83e14a123bc630c2301439ce6182521e5fbafbde88e88ac38314327a4e0493a1bea7e0699a7af808 + languageName: node + linkType: hard + "lines-and-columns@npm:^1.1.6": version: 1.2.4 resolution: "lines-and-columns@npm:1.2.4" @@ -3910,6 +4015,13 @@ __metadata: languageName: node linkType: hard +"pako@npm:~1.0.2": + version: 1.0.11 + resolution: "pako@npm:1.0.11" + checksum: 10c0/86dd99d8b34c3930345b8bbeb5e1cd8a05f608eeb40967b293f72fe469d0e9c88b783a8777e4cc7dc7c91ce54c5e93d88ff4b4f060e6ff18408fd21030d9ffbe + languageName: node + linkType: hard + "param-case@npm:^3.0.4": version: 3.0.4 resolution: "param-case@npm:3.0.4" @@ -4293,7 +4405,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.1": +"readable-stream@npm:^2.0.1, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -4525,6 +4637,18 @@ __metadata: languageName: node linkType: hard +"selenium-webdriver@npm:^4.43.0": + version: 4.43.0 + resolution: "selenium-webdriver@npm:4.43.0" + dependencies: + "@bazel/runfiles": "npm:^6.5.0" + jszip: "npm:^3.10.1" + tmp: "npm:^0.2.5" + ws: "npm:^8.20.0" + checksum: 10c0/3a62794cd7d3c602d537dac886a28917da6bc3a04dcb488aa2b9bd5b8a7d765f705ae561bb7c227350d617fe5c772494632c2d5ee2eea17d887b4b40ee3fac38 + languageName: node + linkType: hard + "selfsigned@npm:^2.4.1": version: 2.4.1 resolution: "selfsigned@npm:2.4.1" @@ -4644,6 +4768,13 @@ __metadata: languageName: node linkType: hard +"setimmediate@npm:^1.0.5": + version: 1.0.5 + resolution: "setimmediate@npm:1.0.5" + checksum: 10c0/5bae81bfdbfbd0ce992893286d49c9693c82b1bcc00dcaaf3a09c8f428fdeacf4190c013598b81875dfac2b08a572422db7df779a99332d0fce186d15a3e4d49 + languageName: node + linkType: hard + "setprototypeof@npm:1.1.0": version: 1.1.0 resolution: "setprototypeof@npm:1.1.0" @@ -5162,6 +5293,13 @@ __metadata: languageName: node linkType: hard +"tmp@npm:^0.2.5": + version: 0.2.5 + resolution: "tmp@npm:0.2.5" + checksum: 10c0/cee5bb7d674bb4ba3ab3f3841c2ca7e46daeb2109eec395c1ec7329a91d52fcb21032b79ac25161a37b2565c4858fefab927af9735926a113ef7bac9091a6e0e + languageName: node + linkType: hard + "to-regex-range@npm:^5.0.1": version: 5.0.1 resolution: "to-regex-range@npm:5.0.1" @@ -5187,6 +5325,15 @@ __metadata: languageName: node linkType: hard +"tree-kill@npm:^1.2.2": + version: 1.2.2 + resolution: "tree-kill@npm:1.2.2" + bin: + tree-kill: cli.js + checksum: 10c0/7b1b7c7f17608a8f8d20a162e7957ac1ef6cd1636db1aba92f4e072dc31818c2ff0efac1e3d91064ede67ed5dc57c565420531a8134090a12ac10cf792ab14d2 + languageName: node + linkType: hard + "ts-loader@npm:^9.5.1": version: 9.5.2 resolution: "ts-loader@npm:9.5.2" @@ -5313,6 +5460,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 + languageName: node + linkType: hard + "unique-filename@npm:^5.0.0": version: 5.0.0 resolution: "unique-filename@npm:5.0.0"