perf(ci): add focused benchmark commands

Marcus Pousette · Marcus Pousette · commit 2e83cb24a6bc · 2026-07-13T12:07:00.000+02:00
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -12,11 +12,12 @@ on:
       suite:
         description: Benchmark suite
         required: true
-        default: full
+        default: quick
         type: choice
         options:
-          - full
+          - quick
           - web
+          - full
 
 permissions:
   contents: read
@@ -28,7 +29,7 @@ jobs:
     if: |
       (github.event_name == 'issue_comment' &&
         github.event.issue.pull_request &&
-        startsWith(github.event.comment.body, '/bench') &&
+        contains(fromJSON('["/bench","/bench quick","/bench web","/bench full"]'), github.event.comment.body) &&
         contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)
       ) || (github.event_name == 'workflow_dispatch')
     runs-on: ubuntu-22.04
@@ -54,10 +55,21 @@ jobs:
               throw new Error('Unable to determine pull request number');
             }
 
+            const commentSuites = {
+              '/bench': 'quick',
+              '/bench quick': 'quick',
+              '/bench web': 'web',
+              '/bench full': 'full',
+            };
+            const commentBody = context.payload.comment?.body?.trim().toLowerCase();
             const suite = context.eventName === 'workflow_dispatch'
-              ? context.payload.inputs?.suite ?? 'full'
-              : 'full';
-            const benchmarkScripts = { full: 'benchmark', web: 'benchmark:web' };
+              ? context.payload.inputs?.suite ?? 'quick'
+              : commentSuites[commentBody];
+            const benchmarkScripts = {
+              quick: 'benchmark:quick',
+              full: 'benchmark',
+              web: 'benchmark:web',
+            };
             if (!Object.hasOwn(benchmarkScripts, suite)) {
               throw new Error(`Unsupported benchmark suite: ${suite}`);
             }
@@ -124,7 +136,6 @@ jobs:
         with:
           ref: ${{ steps.pr.outputs.base_sha }}
           path: bench-base
-          fetch-depth: 0
           submodules: recursive
 
       - name: Checkout head
@@ -133,9 +144,16 @@ jobs:
         with:
           ref: ${{ steps.pr.outputs.head_sha }}
           path: bench-head
-          fetch-depth: 0
           submodules: recursive
 
+      - name: Checkout benchmark tools
+        if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.suite == 'quick'
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.workflow_sha }}
+          path: bench-tools
+          sparse-checkout: scripts
+
       - name: Use Node.js 20.x
         if: steps.pr.outputs.same_repo == 'true'
         uses: actions/setup-node@v4
@@ -151,8 +169,6 @@ jobs:
       - name: Install Rust toolchain
         if: steps.pr.outputs.same_repo == 'true'
         uses: dtolnay/rust-toolchain@stable
-        with:
-          components: rustfmt, clippy
 
       - name: Add wasm targets
         if: steps.pr.outputs.same_repo == 'true'
@@ -219,8 +235,16 @@ jobs:
         run: |
           set -o pipefail
           status=0
-          env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$?
-          pnpm run benchmark:aggregate || status=$?
+          if [ "$BENCHMARK_SCRIPT" = "benchmark:quick" ]; then
+            env -u CI TREECRDT_BENCH_ROOT="$PWD" \
+              node "$GITHUB_WORKSPACE/bench-tools/scripts/run-quick-bench.mjs" \
+              2>&1 | tee benchmark.log || status=$?
+          else
+            env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$?
+          fi
+          if [ ! -f benchmarks/summary.json ]; then
+            pnpm run benchmark:aggregate || status=$?
+          fi
           exit "$status"
 
       - name: Run benchmarks (head)
@@ -234,8 +258,16 @@ jobs:
         run: |
           set -o pipefail
           status=0
-          env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$?
-          pnpm run benchmark:aggregate || status=$?
+          if [ "$BENCHMARK_SCRIPT" = "benchmark:quick" ]; then
+            env -u CI TREECRDT_BENCH_ROOT="$PWD" \
+              node "$GITHUB_WORKSPACE/bench-tools/scripts/run-quick-bench.mjs" \
+              2>&1 | tee benchmark.log || status=$?
+          else
+            env -u CI pnpm run "$BENCHMARK_SCRIPT" 2>&1 | tee benchmark.log || status=$?
+          fi
+          if [ ! -f benchmarks/summary.json ]; then
+            pnpm run benchmark:aggregate || status=$?
+          fi
           exit "$status"
 
       - name: Upload benchmark artifacts
@@ -342,8 +374,9 @@ jobs:
             const baseOutcome = '${{ steps.benchmark_base.outcome }}';
             const headOutcome = '${{ steps.benchmark_head.outcome }}';
             const benchmarkFailed = [baseOutcome, headOutcome].includes('failure');
-            const triggerLabel =
-              context.eventName === 'workflow_dispatch' ? '`workflow_dispatch`' : '`/bench`';
+            const triggerLabel = context.eventName === 'workflow_dispatch'
+              ? '`workflow_dispatch`'
+              : `\`${context.payload.comment.body.trim()}\``;
 
             const parts = [
               marker,
diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md
@@ -14,6 +14,7 @@ pnpm benchmark:sync:help
 Useful top-level entrypoints:
 
 ```sh
+pnpm benchmark:quick
 pnpm benchmark
 pnpm benchmark:sqlite-node
 pnpm benchmark:sqlite-node:ops
@@ -29,7 +30,18 @@ pnpm benchmark:wasm
 pnpm benchmark:postgres
 ```
 
-`pnpm benchmark` writes JSON results under `benchmarks/`.
+Both `pnpm benchmark:quick` and `pnpm benchmark` write JSON results under `benchmarks/`.
+The quick suite samples every primary runtime at a representative scale; the full suite retains the exhaustive size and iteration matrix.
+
+### Pull Request Comparisons
+
+Repository members can trigger base/head comparisons from a pull request:
+
+- `/bench` or `/bench quick`: representative browser/OPFS, native SQLite, direct sync, WASM, and Rust core coverage
+- `/bench web`: full browser memory and OPFS coverage
+- `/bench full`: the exhaustive suite
+
+Use quick for routine feedback and full when confirming a broad or performance-sensitive change.
 
 ## Which Benchmark Answers What?
 
diff --git a/package.json b/package.json
@@ -23,7 +23,8 @@
     "build": "pnpm -r --if-present run build",
     "test:browser": "pnpm -C packages/treecrdt-wa-sqlite/e2e test:e2e",
     "test:native-node": "pnpm -C packages/sync-protocol/material/sqlite run test && pnpm -C packages/treecrdt-sqlite-node run test && pnpm -C packages/treecrdt-wa-sqlite run test",
-    "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sync:direct && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate",
+    "benchmark": "rm -rf benchmarks && mkdir -p benchmarks && pnpm run benchmark:web && pnpm run benchmark:sqlite-node && pnpm run benchmark:wasm && pnpm run benchmark:postgres && pnpm run benchmark:core && pnpm run benchmark:aggregate",
+    "benchmark:quick": "node scripts/run-quick-bench.mjs",
     "benchmark:sqlite-node": "pnpm run benchmark:sqlite-node:ops && pnpm run benchmark:sqlite-node:note-paths && pnpm run benchmark:sqlite-node:sync",
     "benchmark:core": "cargo bench -p treecrdt-core --bench core --features bench",
     "benchmark:aggregate": "node scripts/aggregate-bench.mjs",
diff --git a/packages/treecrdt-benchmark/src/stats.ts b/packages/treecrdt-benchmark/src/stats.ts
@@ -1,5 +1,7 @@
 function readEnv(name: string): string | undefined {
-  const env = (globalThis as any)?.process?.env as Record<string, string | undefined> | undefined;
+  const env = ((globalThis as any)?.process?.env ?? (import.meta as any).env) as
+    | Record<string, string | undefined>
+    | undefined;
   const raw = env?.[name];
   return typeof raw === 'string' && raw.length > 0 ? raw : undefined;
 }
diff --git a/packages/treecrdt-core/benches/core.rs b/packages/treecrdt-core/benches/core.rs
@@ -72,6 +72,7 @@ fn run_benchmark(replica: &ReplicaId, count: u64) -> f64 {
 fn main() {
     let mut out_dir: Option<PathBuf> = None;
     let mut custom_config: Option<Vec<(u64, u64)>> = None;
+    let mut custom_iterations: Option<u64> = None;
     for arg in env::args().skip(1) {
         if let Some(val) = arg.strip_prefix("--count=") {
             let count = val.parse().unwrap_or(500);
@@ -85,17 +86,25 @@ fn main() {
             if !parsed.is_empty() {
                 custom_config = Some(parsed);
             }
+        } else if let Some(val) = arg.strip_prefix("--iterations=") {
+            custom_iterations = val.parse::<u64>().ok().filter(|&iterations| iterations > 0);
         } else if let Some(val) = arg.strip_prefix("--out-dir=") {
             out_dir = Some(PathBuf::from(val));
         }
     }
 
-    let config = custom_config.as_deref().unwrap_or(BENCH_CONFIG);
+    let mut config = custom_config.unwrap_or_else(|| BENCH_CONFIG.to_vec());
+    if let Some(iterations) = custom_iterations {
+        for (_, configured_iterations) in &mut config {
+            *configured_iterations = iterations;
+        }
+    }
+
     let out_dir = out_dir.unwrap_or_else(default_out_dir);
     fs::create_dir_all(&out_dir).expect("mkdirs");
 
     let replica = ReplicaId::new(b"core");
-    for &(count, iterations) in config {
+    for (count, iterations) in config {
         let (duration_ms, iterations_opt, avg_duration_ms) = if iterations > 1 {
             let mut durations: Vec<f64> =
                 (0..iterations).map(|_| run_benchmark(&replica, count)).collect();
diff --git a/packages/treecrdt-wa-sqlite/e2e/vite.config.ts b/packages/treecrdt-wa-sqlite/e2e/vite.config.ts
@@ -15,6 +15,7 @@ const vendorWaSqliteRoot = path.join(vendorPkgRoot, 'wa-sqlite');
 const vendorDistRoot = path.join(vendorPkgRoot, 'dist');
 
 export default defineConfig({
+  envPrefix: ['VITE_', 'BENCH_', 'SYNC_BENCH_'],
   plugins: [
     treecrdtWaSqliteAssets({ outDirs: ['public/wa-sqlite', 'public/base-path/wa-sqlite'] }),
     react(),
diff --git a/scripts/run-quick-bench.mjs b/scripts/run-quick-bench.mjs
@@ -0,0 +1,69 @@
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { repoRootFromImportMeta } from './repo-root.mjs';
+
+const repoRoot = process.env.TREECRDT_BENCH_ROOT
+  ? path.resolve(process.env.TREECRDT_BENCH_ROOT)
+  : repoRootFromImportMeta(import.meta.url, 1);
+const quickEnv = {
+  BENCH_ITERATIONS: '3',
+  BENCH_WARMUP: '1',
+  SYNC_BENCH_SIZES: '1000',
+};
+
+const commands = [
+  ['pnpm', ['run', 'benchmark:web']],
+  ['pnpm', ['-C', 'packages/treecrdt-sqlite-node', 'run', 'benchmark:ops', '--', '--count=1000']],
+  [
+    'pnpm',
+    ['-C', 'packages/treecrdt-sqlite-node', 'run', 'benchmark:note-paths', '--', '--count=10000'],
+  ],
+  [
+    process.execPath,
+    ['scripts/run-sync-bench.mjs', 'direct', '--count=1000', '--iterations=3', '--warmup=1'],
+  ],
+  [
+    'pnpm',
+    [
+      '-C',
+      'packages/treecrdt-wasm-js',
+      'run',
+      'benchmark',
+      '--',
+      '--sizes=1000',
+      '--workloads=insert-move,insert-chain',
+    ],
+  ],
+  [
+    'cargo',
+    [
+      'bench',
+      '-p',
+      'treecrdt-core',
+      '--bench',
+      'core',
+      '--features',
+      'bench',
+      '--',
+      '--count=1000',
+      '--iterations=3',
+    ],
+  ],
+  ['pnpm', ['run', 'benchmark:aggregate']],
+];
+
+fs.rmSync(path.join(repoRoot, 'benchmarks'), { recursive: true, force: true });
+fs.mkdirSync(path.join(repoRoot, 'benchmarks'), { recursive: true });
+
+for (const [command, args] of commands) {
+  console.log(`\n[benchmark:quick] ${command} ${args.join(' ')}`);
+  const result = spawnSync(command, args, {
+    cwd: repoRoot,
+    env: { ...process.env, ...quickEnv },
+    stdio: 'inherit',
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) process.exit(result.status ?? 1);
+}

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,7 @@`
`1`	`1`	`function readEnv(name: string): string \| undefined {`
`2`		`- const env = (globalThis as any)?.process?.env as Record<string, string \| undefined> \| undefined;`
	`2`	`+ const env = ((globalThis as any)?.process?.env ?? (import.meta as any).env) as`
	`3`	`+ \| Record<string, string \| undefined>`
	`4`	`+ \| undefined;`
`3`	`5`	`const raw = env?.[name];`
`4`	`6`	`return typeof raw === 'string' && raw.length > 0 ? raw : undefined;`
`5`	`7`	`}`