From 5d68383e457e3128a827a0d009b14a0a0974cc07 Mon Sep 17 00:00:00 2001 From: RtlZeroMemory <58250858+RtlZeroMemory@users.noreply.github.com> Date: Mon, 16 Feb 2026 12:05:04 +0400 Subject: [PATCH 1/3] Add EPIC-07 CI perf and repro regression gates --- .github/workflows/ci.yml | 52 ++ benchmarks/ci-baseline/config.json | 109 +++ benchmarks/ci-baseline/results.json | 333 +++++++ benchmarks/ci-baseline/results.md | 57 ++ docs/dev/perf-regressions.md | 85 ++ docs/dev/repro-replay.md | 49 ++ docs/dev/testing.md | 4 + package.json | 4 +- packages/bench/profiles/ci.json | 35 + scripts/__tests__/bench-ci-compare.test.mjs | 212 +++++ scripts/bench-ci-compare.mjs | 926 ++++++++++++++++++++ scripts/run-bench-ci.mjs | 379 ++++++++ 12 files changed, 2244 insertions(+), 1 deletion(-) create mode 100644 benchmarks/ci-baseline/config.json create mode 100644 benchmarks/ci-baseline/results.json create mode 100644 benchmarks/ci-baseline/results.md create mode 100644 docs/dev/perf-regressions.md create mode 100644 docs/dev/repro-replay.md create mode 100644 packages/bench/profiles/ci.json create mode 100644 scripts/__tests__/bench-ci-compare.test.mjs create mode 100644 scripts/bench-ci-compare.mjs create mode 100644 scripts/run-bench-ci.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d10a9163..cf133a94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,10 @@ jobs: - name: Tests run: npm run test + - name: Repro replay fixtures (headless, explicit gate) + if: runner.os == 'Linux' + run: node --test --test-concurrency=1 packages/core/dist/repro/__tests__/replay.harness.test.js + - name: Setup Rust (stable) uses: dtolnay/rust-toolchain@stable @@ -152,6 +156,54 @@ jobs: - name: Terminal e2e (reduced) run: bun run test:e2e:reduced + perf-regression: + name: perf regression gate + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout (with submodules) + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + + - name: Install + run: npm ci + + - name: Build packages + run: npm run build + + - name: Run reduced benchmark suite + run: npm run bench:ci -- --output-dir .artifacts/bench/ci + + - name: Compare reduced benchmark results against baseline + run: | + npm run bench:ci:compare -- \ + --baseline benchmarks/ci-baseline/results.json \ + --current .artifacts/bench/ci/results.json \ + --config benchmarks/ci-baseline/config.json \ + --report-json .artifacts/bench/ci/compare.json \ + --report-md .artifacts/bench/ci/compare.md + + - name: Upload perf regression artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: perf-regression-artifacts + if-no-files-found: warn + path: | + .artifacts/bench/ci/results.json + .artifacts/bench/ci/results.md + .artifacts/bench/ci/compare.json + .artifacts/bench/ci/compare.md + .artifacts/bench/ci/manifest.json + .artifacts/bench/ci/profile.json + docs: name: docs runs-on: ubuntu-latest diff --git a/benchmarks/ci-baseline/config.json b/benchmarks/ci-baseline/config.json new file mode 100644 index 00000000..18729d50 --- /dev/null +++ b/benchmarks/ci-baseline/config.json @@ -0,0 +1,109 @@ +{ + "meta": { + "profilePath": "packages/bench/profiles/ci.json", + "baselinePolicy": "Update only on intentional perf shifts or deliberate threshold tuning.", + "baselineGenerationCommand": "npm run bench:ci -- --output-dir benchmarks/ci-baseline", + "comparisonCommandTemplate": "npm run bench:ci:compare -- --current /results.json --report-json /compare.json --report-md /compare.md", + "createdAt": "2026-02-16T06:42:55Z" + }, + "requiredScenarios": [ + { + "scenario": "terminal-rerender", + "framework": "rezi-native", + "params": {} + }, + { + "scenario": "terminal-frame-fill", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 120, + "dirtyLines": 1 + } + }, + { + "scenario": "terminal-frame-fill", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 120, + "dirtyLines": 40 + } + }, + { + "scenario": "terminal-virtual-list", + "framework": "rezi-native", + "params": { + "items": 100000, + "viewport": 40 + } + }, + { + "scenario": "terminal-table", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 8 + } + } + ], + "metrics": { + "defaults": { + "timing.mean": { + "relativeRegression": 0.2, + "absoluteRegression": 0.03, + "severity": "hard", + "direction": "higher_is_worse" + }, + "timing.p95": { + "relativeRegression": 0.35, + "absoluteRegression": 0.1, + "severity": "advisory", + "direction": "higher_is_worse" + } + }, + "overrides": [ + { + "match": { + "scenario": "terminal-frame-fill", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 120, + "dirtyLines": 40 + }, + "metric": "timing.mean" + }, + "rule": { + "relativeRegression": 0.22, + "absoluteRegression": 0.04 + }, + "reason": "Higher dirty-line density has slightly wider spread in CI." + }, + { + "match": { + "scenario": "terminal-virtual-list", + "framework": "rezi-native", + "metric": "timing.mean" + }, + "rule": { + "relativeRegression": 0.22, + "absoluteRegression": 0.06 + }, + "reason": "Large-list workload has greater host scheduler sensitivity." + }, + { + "match": { + "scenario": "terminal-virtual-list", + "framework": "rezi-native", + "metric": "timing.p95" + }, + "rule": { + "relativeRegression": 0.4, + "absoluteRegression": 0.12 + }, + "reason": "Tail latency is noisier for this scenario on shared CI hosts." + } + ] + } +} diff --git a/benchmarks/ci-baseline/results.json b/benchmarks/ci-baseline/results.json new file mode 100644 index 00000000..31987728 --- /dev/null +++ b/benchmarks/ci-baseline/results.json @@ -0,0 +1,333 @@ +{ + "meta": { + "timestamp": "2026-02-16T06:57:58.155Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64", + "osType": "Linux", + "osRelease": "6.6.87.2-microsoft-standard-WSL2", + "cpuModel": "AMD Ryzen 7 9800X3D 8-Core Processor", + "cpuCores": 12, + "memoryTotalMb": 15993 + }, + "invocation": { + "suite": "terminal", + "scenarioFilter": null, + "frameworkFilter": "rezi-native", + "iterationsOverride": 800, + "warmupOverride": 50, + "quick": false, + "ioMode": "stub" + }, + "results": [ + { + "scenario": "terminal-rerender", + "framework": "rezi-native", + "params": {}, + "metrics": { + "timing": { + "n": 800, + "mean": 0.025910159999999766, + "median": 0.016370999999992364, + "p95": 0.0790140000000008, + "p99": 0.11780600000000163, + "min": 0.00970999999998412, + "max": 0.42496099999999615, + "stddev": 0.034548499526925215, + "cv": 1.3333958388109346, + "meanCi95Low": 0.023927173750000003, + "meanCi95High": 0.028436486249999095 + }, + "memBefore": { + "rssKb": 60324, + "heapUsedKb": 8345, + "heapTotalKb": 17448, + "externalKb": 1816, + "arrayBuffersKb": 44 + }, + "memAfter": { + "rssKb": 65532, + "heapUsedKb": 10767, + "heapTotalKb": 19240, + "externalKb": 1816, + "arrayBuffersKb": 44 + }, + "memPeak": { + "rssKb": 65532, + "heapUsedKb": 13549, + "heapTotalKb": 19240, + "externalKb": 1816, + "arrayBuffersKb": 44 + }, + "rssGrowthKb": 5208, + "heapUsedGrowthKb": 2422, + "rssSlopeKbPerIter": null, + "heapUsedSlopeKbPerIter": null, + "memStable": null, + "cpu": { + "userMs": 47.138000000000005, + "systemMs": 2.830000000000002 + }, + "iterations": 800, + "totalWallMs": 21.124217000000016, + "opsPerSec": 37871.22618556699, + "framesProduced": 800, + "bytesProduced": 185652 + }, + "timestamp": "2026-02-16T06:57:58.271Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64" + }, + { + "scenario": "terminal-frame-fill", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 120, + "dirtyLines": 1 + }, + "metrics": { + "timing": { + "n": 800, + "mean": 0.04680949749999994, + "median": 0.03033600000000547, + "p95": 0.11275600000000452, + "p99": 0.30505500000001007, + "min": 0.02400200000002428, + "max": 0.48030400000001805, + "stddev": 0.04769691433552194, + "cv": 1.0189580508853358, + "meanCi95Low": 0.04411405750000012, + "meanCi95High": 0.049964247500000204 + }, + "memBefore": { + "rssKb": 64416, + "heapUsedKb": 8507, + "heapTotalKb": 17960, + "externalKb": 1828, + "arrayBuffersKb": 57 + }, + "memAfter": { + "rssKb": 77060, + "heapUsedKb": 25188, + "heapTotalKb": 30248, + "externalKb": 1836, + "arrayBuffersKb": 64 + }, + "memPeak": { + "rssKb": 77060, + "heapUsedKb": 25188, + "heapTotalKb": 30248, + "externalKb": 1836, + "arrayBuffersKb": 64 + }, + "rssGrowthKb": 12644, + "heapUsedGrowthKb": 16681, + "rssSlopeKbPerIter": null, + "heapUsedSlopeKbPerIter": null, + "memStable": null, + "cpu": { + "userMs": 62.85900000000001, + "systemMs": 14.193000000000005 + }, + "iterations": 800, + "totalWallMs": 37.875388, + "opsPerSec": 21121.89583377997, + "framesProduced": 800, + "bytesProduced": 275200 + }, + "timestamp": "2026-02-16T06:57:58.417Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64" + }, + { + "scenario": "terminal-frame-fill", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 120, + "dirtyLines": 40 + }, + "metrics": { + "timing": { + "n": 800, + "mean": 0.11282477749999989, + "median": 0.09059450000000879, + "p95": 0.2054510000000107, + "p99": 0.45930300000000557, + "min": 0.06829400000000874, + "max": 0.9811590000000194, + "stddev": 0.07118812704328271, + "cv": 0.630961820804679, + "meanCi95Low": 0.10867026499999985, + "meanCi95High": 0.11813537499999936 + }, + "memBefore": { + "rssKb": 64304, + "heapUsedKb": 8864, + "heapTotalKb": 17960, + "externalKb": 1939, + "arrayBuffersKb": 168 + }, + "memAfter": { + "rssKb": 81376, + "heapUsedKb": 14866, + "heapTotalKb": 32552, + "externalKb": 1904, + "arrayBuffersKb": 133 + }, + "memPeak": { + "rssKb": 87648, + "heapUsedKb": 26593, + "heapTotalKb": 39976, + "externalKb": 2235, + "arrayBuffersKb": 463 + }, + "rssGrowthKb": 17072, + "heapUsedGrowthKb": 6002, + "rssSlopeKbPerIter": null, + "heapUsedSlopeKbPerIter": null, + "memStable": null, + "cpu": { + "userMs": 131.20600000000002, + "systemMs": 17.587999999999997 + }, + "iterations": 800, + "totalWallMs": 90.722006, + "opsPerSec": 8818.147164867585, + "framesProduced": 800, + "bytesProduced": 5747200 + }, + "timestamp": "2026-02-16T06:57:58.585Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64" + }, + { + "scenario": "terminal-virtual-list", + "framework": "rezi-native", + "params": { + "items": 100000, + "viewport": 40 + }, + "metrics": { + "timing": { + "n": 800, + "mean": 0.26352908125000024, + "median": 0.22015100000001553, + "p95": 0.6459629999999947, + "p99": 0.9262669999999957, + "min": 0.17140899999998283, + "max": 1.5334569999999985, + "stddev": 0.15578726030507803, + "cv": 0.5911577559718673, + "meanCi95Low": 0.2545057762499992, + "meanCi95High": 0.27491137999999965 + }, + "memBefore": { + "rssKb": 83656, + "heapUsedKb": 9458, + "heapTotalKb": 26664, + "externalKb": 1820, + "arrayBuffersKb": 48 + }, + "memAfter": { + "rssKb": 128284, + "heapUsedKb": 29371, + "heapTotalKb": 79656, + "externalKb": 1820, + "arrayBuffersKb": 49 + }, + "memPeak": { + "rssKb": 136332, + "heapUsedKb": 69042, + "heapTotalKb": 88104, + "externalKb": 1820, + "arrayBuffersKb": 49 + }, + "rssGrowthKb": 44628, + "heapUsedGrowthKb": 19913, + "rssSlopeKbPerIter": null, + "heapUsedSlopeKbPerIter": null, + "memStable": null, + "cpu": { + "userMs": 267.14699999999993, + "systemMs": 29.964000000000002 + }, + "iterations": 800, + "totalWallMs": 211.489891, + "opsPerSec": 3782.6867100706954, + "framesProduced": 800, + "bytesProduced": 3915584 + }, + "timestamp": "2026-02-16T06:57:58.822Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64" + }, + { + "scenario": "terminal-table", + "framework": "rezi-native", + "params": { + "rows": 40, + "cols": 8 + }, + "metrics": { + "timing": { + "n": 800, + "mean": 0.0668960075000001, + "median": 0.04199199999999337, + "p95": 0.1597680000000139, + "p99": 0.4018700000000024, + "min": 0.029821999999995796, + "max": 0.9898000000000025, + "stddev": 0.07284240508409222, + "cv": 1.0888901715710328, + "meanCi95Low": 0.06280069249999957, + "meanCi95High": 0.07206160375000088 + }, + "memBefore": { + "rssKb": 63776, + "heapUsedKb": 8735, + "heapTotalKb": 17704, + "externalKb": 1830, + "arrayBuffersKb": 58 + }, + "memAfter": { + "rssKb": 77956, + "heapUsedKb": 15193, + "heapTotalKb": 35624, + "externalKb": 1889, + "arrayBuffersKb": 118 + }, + "memPeak": { + "rssKb": 77956, + "heapUsedKb": 23596, + "heapTotalKb": 37416, + "externalKb": 1889, + "arrayBuffersKb": 118 + }, + "rssGrowthKb": 14180, + "heapUsedGrowthKb": 6458, + "rssSlopeKbPerIter": null, + "heapUsedSlopeKbPerIter": null, + "memStable": null, + "cpu": { + "userMs": 107.134, + "systemMs": 12.023000000000003 + }, + "iterations": 800, + "totalWallMs": 53.926028, + "opsPerSec": 14835.136754370264, + "framesProduced": 800, + "bytesProduced": 660640 + }, + "timestamp": "2026-02-16T06:57:59.221Z", + "nodeVersion": "v20.19.5", + "platform": "linux", + "arch": "x64" + } + ] +} diff --git a/benchmarks/ci-baseline/results.md b/benchmarks/ci-baseline/results.md new file mode 100644 index 00000000..efb6917e --- /dev/null +++ b/benchmarks/ci-baseline/results.md @@ -0,0 +1,57 @@ +# Benchmark Results + +> 2026-02-16T06:57:58.155Z | Node v20.19.5 | Linux 6.6.87.2-microsoft-standard-WSL2 | linux x64 | AMD Ryzen 7 9800X3D 8-Core Processor (12 cores) | RAM 15993MB + +> Invocation: suite=terminal scenario=all framework=rezi-native warmup=50 iterations=800 quick=no io=stub + +## terminal-rerender + +| Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Rezi (native) | 26µs | 35µs | 24µs–28µs | 37.9K ops/s | 21.12ms | 47.14ms | 2.83ms | 64.0MB | 13.2MB | 181.30078125KB | + +## terminal-frame-fill (rows=40, cols=120, dirtyLines=1) + +| Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Rezi (native) | 47µs | 48µs | 44µs–50µs | 21.1K ops/s | 37.88ms | 62.86ms | 14.19ms | 75.3MB | 24.6MB | 268.75KB | + +## terminal-frame-fill (rows=40, cols=120, dirtyLines=40) + +| Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Rezi (native) | 113µs | 71µs | 109µs–118µs | 8.8K ops/s | 90.72ms | 131.21ms | 17.59ms | 85.6MB | 26.0MB | 5.5MB | + +## terminal-virtual-list (items=100000, viewport=40) + +| Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Rezi (native) | 264µs | 156µs | 255µs–275µs | 3.8K ops/s | 211.49ms | 267.15ms | 29.96ms | 133.1MB | 67.4MB | 3.7MB | + +## terminal-table (rows=40, cols=8) + +| Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| Rezi (native) | 67µs | 73µs | 63µs–72µs | 14.8K ops/s | 53.93ms | 107.13ms | 12.02ms | 76.1MB | 23.0MB | 645.15625KB | + +## Relative Performance (vs Rezi native) + +> "Xx slower" = Rezi native is X times faster. "Xx faster" = other framework is faster. + +| Scenario | | +|---|| +| terminal-rerender | | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=1) | | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=40) | | +| terminal-virtual-list (items=100000, viewport=40) | | +| terminal-table (rows=40, cols=8) | | + +## Memory Comparison + +| Scenario | Framework | Peak RSS | Peak Heap | RSS Growth | Heap Growth | RSS Slope | Stable | +|---|---|---:|---:|---:|---:|---:|---:| +| terminal-rerender | Rezi (native) | 64.0MB | 13.2MB | +5.1MB | +2.4MB | N/A | N/A | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=1) | Rezi (native) | 75.3MB | 24.6MB | +12.3MB | +16.3MB | N/A | N/A | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=40) | Rezi (native) | 85.6MB | 26.0MB | +16.7MB | +5.9MB | N/A | N/A | +| terminal-virtual-list (items=100000, viewport=40) | Rezi (native) | 133.1MB | 67.4MB | +43.6MB | +19.4MB | N/A | N/A | +| terminal-table (rows=40, cols=8) | Rezi (native) | 76.1MB | 23.0MB | +13.8MB | +6.3MB | N/A | N/A | diff --git a/docs/dev/perf-regressions.md b/docs/dev/perf-regressions.md new file mode 100644 index 00000000..9869d1f8 --- /dev/null +++ b/docs/dev/perf-regressions.md @@ -0,0 +1,85 @@ +# Perf Regressions + +The CI performance gate protects the terminal rendering differentiators from silent regressions. + +CI workflow: `.github/workflows/ci.yml` +Job name: `perf regression gate` + +## PR Gate Scope + +The gate runs a reduced, deterministic benchmark profile: + +- suite: `terminal` +- framework: `rezi-native` +- io mode: `stub` (default for CI determinism) +- required scenarios: + - `terminal-rerender` + - `terminal-frame-fill` (`dirtyLines=1` and `dirtyLines=40`) + - `terminal-virtual-list` + - `terminal-table` + +Profile source: `packages/bench/profiles/ci.json` + +## Local Commands + +Build first (bench runner lives in `packages/bench/dist`): + +```bash +npm ci +npm run build +``` + +Run the reduced profile: + +```bash +npm run bench:ci -- --output-dir .artifacts/bench/ci-local +``` + +Compare to baseline: + +```bash +npm run bench:ci:compare -- \ + --baseline benchmarks/ci-baseline/results.json \ + --current .artifacts/bench/ci-local/results.json \ + --config benchmarks/ci-baseline/config.json \ + --report-json .artifacts/bench/ci-local/compare.json \ + --report-md .artifacts/bench/ci-local/compare.md +``` + +## Threshold Model + +Threshold config: `benchmarks/ci-baseline/config.json` + +Rules are `max(relative_regression * baseline, absolute_regression)` per metric: + +- `timing.mean` is a **hard gate** (fails CI when exceeded). +- `timing.p95` is **advisory** by default (recorded in report, non-fatal). + +This dual threshold prevents false failures from tiny absolute timing shifts on very fast scenarios, while still failing meaningful slowdowns. + +## Artifacts + +The CI job uploads artifacts on both success and failure: + +- `.artifacts/bench/ci/results.json` +- `.artifacts/bench/ci/results.md` +- `.artifacts/bench/ci/compare.json` +- `.artifacts/bench/ci/compare.md` +- `.artifacts/bench/ci/manifest.json` +- `.artifacts/bench/ci/profile.json` + +`compare.json` and `compare.md` include actionable rows with scenario, metric, baseline, current, delta, and threshold. + +## Intentional Baseline Updates + +Baseline path: + +- `benchmarks/ci-baseline/results.json` +- `benchmarks/ci-baseline/results.md` + +Update process: + +1. Run `npm run bench:ci -- --output-dir benchmarks/ci-baseline`. +2. Re-run compare against the updated baseline to verify zero hard regressions. +3. Keep threshold changes (`benchmarks/ci-baseline/config.json`) explicit in the same PR only when justified. +4. In PR description, include why baseline movement is intentional (feature/perf tradeoff, infra change, or expected engine shift). diff --git a/docs/dev/repro-replay.md b/docs/dev/repro-replay.md new file mode 100644 index 00000000..0a0e9434 --- /dev/null +++ b/docs/dev/repro-replay.md @@ -0,0 +1,49 @@ +# Repro Replay + +Repro replay fixtures validate deterministic headless record/replay behavior. + +## Fixture Source + +- replay fixtures: `packages/testkit/fixtures/repro/*.json` +- primary replay harness test: `packages/core/src/repro/__tests__/replay.harness.test.ts` + +The fixture currently exercised by the harness is: + +- `packages/testkit/fixtures/repro/replay_resize_tab_text.json` + +## CI Gate + +Every PR runs an explicit replay gate in `.github/workflows/ci.yml`: + +- job: `node / ubuntu-latest` +- step: `Repro replay fixtures (headless, explicit gate)` +- command: + +```bash +node --test --test-concurrency=1 packages/core/dist/repro/__tests__/replay.harness.test.js +``` + +`npm run test` also covers this area, but this dedicated step keeps replay coverage visible as a standalone gate. + +## Local Run + +```bash +npm ci +npm run build +node --test --test-concurrency=1 packages/core/dist/repro/__tests__/replay.harness.test.js +``` + +Optional schema/version checks: + +```bash +node --test --test-concurrency=1 packages/core/dist/repro/__tests__/schema.versioning.test.js +``` + +## Updating Replay Fixtures + +Follow the golden fixture policy in `packages/testkit/fixtures/README.md`: + +1. Update fixture JSON intentionally (no auto-regeneration in committed tests). +2. Re-run replay harness and related tests. +3. Include fixture diff context and reasoning in the PR. + diff --git a/docs/dev/testing.md b/docs/dev/testing.md index 65cc290b..4b471903 100644 --- a/docs/dev/testing.md +++ b/docs/dev/testing.md @@ -12,3 +12,7 @@ The repo uses: - golden tests for drawlists/layout/routing where byte-level stability matters - fuzz-lite tests for binary parsers (bounded, never-throw) +Related CI gates: + +- [Perf Regressions](./perf-regressions.md) +- [Repro Replay](./repro-replay.md) diff --git a/package.json b/package.json index 32bb8d95..83f24c1b 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,9 @@ "test:scripts": "node scripts/run-tests.mjs --scope scripts", "test:packages": "node scripts/run-tests.mjs --scope packages", "bench": "node --expose-gc packages/bench/dist/run.js", - "bench:report": "node --expose-gc packages/bench/dist/run.js --markdown" + "bench:report": "node --expose-gc packages/bench/dist/run.js --markdown", + "bench:ci": "node scripts/run-bench-ci.mjs", + "bench:ci:compare": "node scripts/bench-ci-compare.mjs" }, "devDependencies": { "@biomejs/biome": "^1.9.4", diff --git a/packages/bench/profiles/ci.json b/packages/bench/profiles/ci.json new file mode 100644 index 00000000..6fc43415 --- /dev/null +++ b/packages/bench/profiles/ci.json @@ -0,0 +1,35 @@ +{ + "name": "ci", + "description": "Reduced deterministic benchmark profile for PR regression gating.", + "suite": "terminal", + "requiredScenarios": [ + "terminal-rerender", + "terminal-frame-fill", + "terminal-virtual-list", + "terminal-table" + ], + "requiredScenarioParams": { + "terminal-rerender": [{}], + "terminal-frame-fill": [ + { "rows": 40, "cols": 120, "dirtyLines": 1 }, + { "rows": 40, "cols": 120, "dirtyLines": 40 } + ], + "terminal-virtual-list": [{ "items": 100000, "viewport": 40 }], + "terminal-table": [{ "rows": 40, "cols": 8 }] + }, + "framework": "rezi-native", + "io": "stub", + "warmup": 50, + "iterations": 800, + "outputDir": "out/bench-ci", + "output": { + "resultsJson": "results.json", + "resultsMarkdown": "results.md" + }, + "deterministicOrdering": { + "execution": "sequential", + "scenarioOrder": "scenario-registry-order filtered by --suite terminal", + "frameworkOrder": "fixed-single-framework", + "paramSetOrder": "as-defined-in-scenario-source" + } +} diff --git a/scripts/__tests__/bench-ci-compare.test.mjs b/scripts/__tests__/bench-ci-compare.test.mjs new file mode 100644 index 00000000..481e0377 --- /dev/null +++ b/scripts/__tests__/bench-ci-compare.test.mjs @@ -0,0 +1,212 @@ +/** + * Tests for bench-ci-compare.mjs + */ + +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { compareBenchRuns, normalizeComparatorConfig } from "../bench-ci-compare.mjs"; + +function makeConfig(overrides = []) { + return normalizeComparatorConfig({ + requiredScenarios: [ + { + scenario: "terminal-rerender", + framework: "rezi-native", + params: {}, + }, + { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 1 }, + }, + { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 40 }, + }, + ], + metrics: { + defaults: { + "timing.mean": { + relativeRegression: 0.05, + absoluteRegression: 0.02, + severity: "hard", + direction: "higher_is_worse", + }, + "timing.p95": { + relativeRegression: 0.1, + absoluteRegression: 0.05, + severity: "advisory", + direction: "higher_is_worse", + }, + }, + overrides, + }, + }); +} + +function makeRun(meanByKey, p95ByKey) { + const results = []; + for (const key of Object.keys(meanByKey)) { + const [scenario, paramsJson] = key.split("|"); + const params = JSON.parse(paramsJson); + results.push({ + scenario, + framework: "rezi-native", + params, + metrics: { + timing: { + mean: meanByKey[key], + p95: p95ByKey[key], + }, + }, + }); + } + return { results }; +} + +const KEY_RERENDER = "terminal-rerender|{}"; +const KEY_FILL_1 = 'terminal-frame-fill|{"cols":120,"dirtyLines":1,"rows":40}'; +const KEY_FILL_40 = 'terminal-frame-fill|{"cols":120,"dirtyLines":40,"rows":40}'; + +describe("bench-ci-compare", () => { + test("fails on hard mean regression", () => { + const config = makeConfig(); + const baseline = makeRun( + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + { + [KEY_RERENDER]: 1.2, + [KEY_FILL_1]: 1.2, + [KEY_FILL_40]: 1.2, + }, + ); + const current = makeRun( + { + [KEY_RERENDER]: 1.25, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + { + [KEY_RERENDER]: 1.2, + [KEY_FILL_1]: 1.2, + [KEY_FILL_40]: 1.2, + }, + ); + + const report = compareBenchRuns(baseline, current, config); + assert.equal(report.summary.hardRegressions, 1); + assert.equal(report.summary.exitCode, 1); + }); + + test("advisory p95 regression does not fail exit code", () => { + const config = makeConfig(); + const baseline = makeRun( + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + ); + const current = makeRun( + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + { + [KEY_RERENDER]: 1.2, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + ); + + const report = compareBenchRuns(baseline, current, config); + assert.equal(report.summary.hardRegressions, 0); + assert.equal(report.summary.advisoryRegressions, 1); + assert.equal(report.summary.exitCode, 0); + }); + + test("supports per-scenario metric overrides", () => { + const config = makeConfig([ + { + match: { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 40 }, + metric: "timing.mean", + }, + rule: { + relativeRegression: 0.3, + absoluteRegression: 0.2, + }, + }, + ]); + + const baseline = makeRun( + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + ); + const current = makeRun( + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1.25, + }, + { + [KEY_RERENDER]: 1, + [KEY_FILL_1]: 1, + [KEY_FILL_40]: 1, + }, + ); + + const report = compareBenchRuns(baseline, current, config); + assert.equal(report.summary.hardRegressions, 0); + assert.equal(report.summary.exitCode, 0); + }); + + test("requires both terminal-frame-fill variants in config", () => { + assert.throws( + () => + normalizeComparatorConfig({ + requiredScenarios: [ + { + scenario: "terminal-rerender", + framework: "rezi-native", + params: {}, + }, + { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 1 }, + }, + ], + metrics: { + defaults: { + "timing.mean": { + relativeRegression: 0.1, + absoluteRegression: 0.1, + }, + }, + }, + }), + /requiredScenarios must explicitly include terminal-frame-fill variants/, + ); + }); +}); diff --git a/scripts/bench-ci-compare.mjs b/scripts/bench-ci-compare.mjs new file mode 100644 index 00000000..d46134f9 --- /dev/null +++ b/scripts/bench-ci-compare.mjs @@ -0,0 +1,926 @@ +#!/usr/bin/env node +/** + * CI benchmark comparator for regression gating. + * + * Compares a "current" bench run JSON against a committed baseline JSON using + * per-metric threshold rules from config. Emits: + * - machine-readable JSON report + * - markdown summary report + * + * Exit codes: + * - 0: no hard regressions + * - 1: hard regressions, missing required comparisons, or invalid input + */ + +import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const DEFAULT_BASELINE_PATH = join(ROOT, "benchmarks", "ci-baseline", "results.json"); +const DEFAULT_CONFIG_PATH = join(ROOT, "benchmarks", "ci-baseline", "config.json"); +const DEFAULT_CURRENT_PATH = join(ROOT, "out", "bench-ci", "results.json"); +const DEFAULT_REPORT_JSON_PATH = join(ROOT, "out", "bench-ci", "compare.json"); +const DEFAULT_REPORT_MD_PATH = join(ROOT, "out", "bench-ci", "compare.md"); + +const STATUS = Object.freeze({ + PASS: "pass", + HARD_REGRESSION: "hard_regression", + ADVISORY_REGRESSION: "advisory_regression", + MISSING_BASELINE_RESULT: "missing_baseline_result", + MISSING_CURRENT_RESULT: "missing_current_result", + MISSING_BASELINE_METRIC: "missing_baseline_metric", + MISSING_CURRENT_METRIC: "missing_current_metric", +}); + +const HARD_STATUSES = new Set([ + STATUS.HARD_REGRESSION, + STATUS.MISSING_BASELINE_RESULT, + STATUS.MISSING_CURRENT_RESULT, + STATUS.MISSING_BASELINE_METRIC, + STATUS.MISSING_CURRENT_METRIC, +]); + +const REQUIRED_FRAME_FILL_VARIANTS = Object.freeze([ + { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 1 }, + }, + { + scenario: "terminal-frame-fill", + framework: "rezi-native", + params: { rows: 40, cols: 120, dirtyLines: 40 }, + }, +]); + +function die(message) { + process.stderr.write(`${message}\n`); + process.exit(1); +} + +function parseArgs(argv) { + const opts = { + baselinePath: DEFAULT_BASELINE_PATH, + currentPath: DEFAULT_CURRENT_PATH, + configPath: DEFAULT_CONFIG_PATH, + reportJsonPath: DEFAULT_REPORT_JSON_PATH, + reportMdPath: DEFAULT_REPORT_MD_PATH, + failOnAdvisory: false, + }; + + for (let i = 2; i < argv.length; i++) { + const arg = argv[i]; + switch (arg) { + case "--baseline": + opts.baselinePath = argv[++i] ?? null; + break; + case "--current": + opts.currentPath = argv[++i] ?? null; + break; + case "--config": + opts.configPath = argv[++i] ?? null; + break; + case "--report-json": + opts.reportJsonPath = argv[++i] ?? null; + break; + case "--report-md": + opts.reportMdPath = argv[++i] ?? null; + break; + case "--fail-on-advisory": + opts.failOnAdvisory = true; + break; + case "--help": + case "-h": + printUsage(); + process.exit(0); + break; + default: + die(`bench-ci-compare: unknown argument: ${arg}`); + } + } + + if (!opts.baselinePath || !opts.configPath || !opts.reportJsonPath || !opts.reportMdPath) { + die("bench-ci-compare: --baseline, --config, --report-json, and --report-md require values"); + } + + return opts; +} + +function printUsage() { + process.stdout.write( + [ + "Usage:", + " node scripts/bench-ci-compare.mjs [options]", + "", + "Options:", + ` --current Current results JSON (default: ${toRel(DEFAULT_CURRENT_PATH)})`, + ` --baseline Baseline results JSON (default: ${toRel(DEFAULT_BASELINE_PATH)})`, + ` --config Threshold config JSON (default: ${toRel(DEFAULT_CONFIG_PATH)})`, + ` --report-json Machine report output (default: ${toRel(DEFAULT_REPORT_JSON_PATH)})`, + ` --report-md Markdown report output (default: ${toRel(DEFAULT_REPORT_MD_PATH)})`, + " --fail-on-advisory Also fail CI when advisory regressions are present", + " --help, -h Show this help", + "", + ].join("\n"), + ); +} + +function readJson(path, label) { + let raw; + try { + raw = readFileSync(path, "utf8"); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + throw new Error(`${label}: failed to read ${path}\n${detail}`); + } + + try { + return JSON.parse(raw); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + throw new Error(`${label}: failed to parse JSON ${path}\n${detail}`); + } +} + +function isPlainObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeParams(value, source) { + if (!isPlainObject(value)) { + throw new Error(`${source} must be an object`); + } + const out = {}; + const keys = Object.keys(value).sort(); + for (const key of keys) { + const v = value[key]; + if (typeof v !== "number" && typeof v !== "string" && typeof v !== "boolean") { + throw new Error( + `${source}.${key} must be number|string|boolean (got ${v === null ? "null" : typeof v})`, + ); + } + out[key] = v; + } + return out; +} + +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map((v) => stableStringify(v)).join(",")}]`; + } + if (isPlainObject(value)) { + const keys = Object.keys(value).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function makeDescriptorKey(scenario, framework, params) { + return `${scenario}|${framework}|${stableStringify(params ?? {})}`; +} + +function normalizeScenarioDescriptor(raw, source) { + if (!isPlainObject(raw)) { + throw new Error(`${source} must be an object`); + } + + const scenario = raw.scenario; + const framework = raw.framework; + if (typeof scenario !== "string" || scenario.length === 0) { + throw new Error(`${source}.scenario must be a non-empty string`); + } + if (typeof framework !== "string" || framework.length === 0) { + throw new Error(`${source}.framework must be a non-empty string`); + } + + const params = normalizeParams(raw.params ?? {}, `${source}.params`); + + return { + scenario, + framework, + params, + key: makeDescriptorKey(scenario, framework, params), + }; +} + +function normalizeSeverity(raw, source) { + if (raw === undefined) return undefined; + if (raw === "hard" || raw === "advisory") return raw; + throw new Error(`${source} must be "hard" or "advisory"`); +} + +function normalizeDirection(raw, source) { + if (raw === undefined) return undefined; + if (raw === "higher_is_worse" || raw === "lower_is_worse") return raw; + throw new Error(`${source} must be "higher_is_worse" or "lower_is_worse"`); +} + +function normalizeNumber(raw, source) { + const n = Number(raw); + if (!Number.isFinite(n) || n < 0) { + throw new Error(`${source} must be a non-negative finite number`); + } + return n; +} + +function normalizeDefaultRule(metric, raw, source) { + if (!isPlainObject(raw)) { + throw new Error(`${source} must be an object`); + } + return { + metric, + relativeRegression: normalizeNumber(raw.relativeRegression, `${source}.relativeRegression`), + absoluteRegression: normalizeNumber(raw.absoluteRegression, `${source}.absoluteRegression`), + severity: normalizeSeverity(raw.severity, `${source}.severity`) ?? "hard", + direction: normalizeDirection(raw.direction, `${source}.direction`) ?? "higher_is_worse", + source, + }; +} + +function normalizeOverride(raw, index, defaultRules) { + const source = `metrics.overrides[${index}]`; + if (!isPlainObject(raw)) { + throw new Error(`${source} must be an object`); + } + if (!isPlainObject(raw.match)) { + throw new Error(`${source}.match must be an object`); + } + if (!isPlainObject(raw.rule)) { + throw new Error(`${source}.rule must be an object`); + } + + const metric = raw.match.metric; + if (typeof metric !== "string" || metric.length === 0) { + throw new Error(`${source}.match.metric must be a non-empty string`); + } + if (!defaultRules.has(metric)) { + throw new Error( + `${source}.match.metric=${metric} has no matching default rule in metrics.defaults`, + ); + } + + const match = { + metric, + scenario: + raw.match.scenario === undefined + ? undefined + : validateString(raw.match.scenario, `${source}.match.scenario`), + framework: + raw.match.framework === undefined + ? undefined + : validateString(raw.match.framework, `${source}.match.framework`), + params: + raw.match.params === undefined + ? undefined + : normalizeParams(raw.match.params, `${source}.match.params`), + }; + + const rulePatch = { + relativeRegression: + raw.rule.relativeRegression === undefined + ? undefined + : normalizeNumber(raw.rule.relativeRegression, `${source}.rule.relativeRegression`), + absoluteRegression: + raw.rule.absoluteRegression === undefined + ? undefined + : normalizeNumber(raw.rule.absoluteRegression, `${source}.rule.absoluteRegression`), + severity: normalizeSeverity(raw.rule.severity, `${source}.rule.severity`), + direction: normalizeDirection(raw.rule.direction, `${source}.rule.direction`), + }; + + if ( + rulePatch.relativeRegression === undefined && + rulePatch.absoluteRegression === undefined && + rulePatch.severity === undefined && + rulePatch.direction === undefined + ) { + throw new Error(`${source}.rule must include at least one override field`); + } + + const reason = raw.reason === undefined ? null : validateString(raw.reason, `${source}.reason`); + + return { source, match, rulePatch, reason }; +} + +function validateString(raw, source) { + if (typeof raw !== "string" || raw.length === 0) { + throw new Error(`${source} must be a non-empty string`); + } + return raw; +} + +function assertRequiredFrameFillVariants(requiredScenarios) { + const keys = new Set(requiredScenarios.map((d) => d.key)); + for (const required of REQUIRED_FRAME_FILL_VARIANTS) { + const key = makeDescriptorKey(required.scenario, required.framework, required.params); + if (!keys.has(key)) { + throw new Error( + [ + "requiredScenarios must explicitly include terminal-frame-fill variants:", + "- scenario=terminal-frame-fill framework=rezi-native params={rows:40,cols:120,dirtyLines:1}", + "- scenario=terminal-frame-fill framework=rezi-native params={rows:40,cols:120,dirtyLines:40}", + ].join("\n"), + ); + } + } +} + +export function normalizeComparatorConfig(rawConfig) { + if (!isPlainObject(rawConfig)) { + throw new Error("config root must be an object"); + } + + const requiredRaw = rawConfig.requiredScenarios; + if (!Array.isArray(requiredRaw) || requiredRaw.length === 0) { + throw new Error("config.requiredScenarios must be a non-empty array"); + } + + const requiredScenarios = requiredRaw.map((entry, index) => + normalizeScenarioDescriptor(entry, `requiredScenarios[${index}]`), + ); + + const seenKeys = new Set(); + for (const entry of requiredScenarios) { + if (seenKeys.has(entry.key)) { + throw new Error(`duplicate required scenario entry: ${entry.key}`); + } + seenKeys.add(entry.key); + } + + assertRequiredFrameFillVariants(requiredScenarios); + + if (!isPlainObject(rawConfig.metrics)) { + throw new Error("config.metrics must be an object"); + } + if (!isPlainObject(rawConfig.metrics.defaults)) { + throw new Error("config.metrics.defaults must be an object"); + } + + const defaultRuleEntries = Object.entries(rawConfig.metrics.defaults).sort((a, b) => + a[0].localeCompare(b[0]), + ); + if (defaultRuleEntries.length === 0) { + throw new Error("config.metrics.defaults must define at least one metric rule"); + } + + const defaultRules = new Map(); + for (const [metric, ruleRaw] of defaultRuleEntries) { + if (typeof metric !== "string" || metric.length === 0) { + throw new Error("metrics.defaults keys must be non-empty metric paths"); + } + const normalized = normalizeDefaultRule(metric, ruleRaw, `metrics.defaults.${metric}`); + defaultRules.set(metric, normalized); + } + + const overridesRaw = rawConfig.metrics.overrides ?? []; + if (!Array.isArray(overridesRaw)) { + throw new Error("config.metrics.overrides must be an array when provided"); + } + const overrides = overridesRaw.map((entry, index) => + normalizeOverride(entry, index, defaultRules), + ); + + return { + meta: isPlainObject(rawConfig.meta) ? rawConfig.meta : null, + requiredScenarios, + defaultRules, + overrides, + }; +} + +function normalizeBenchRun(rawRun, source) { + if (!isPlainObject(rawRun)) { + throw new Error(`${source} root must be an object`); + } + if (!Array.isArray(rawRun.results)) { + throw new Error(`${source}.results must be an array`); + } + return rawRun; +} + +function indexBenchResults(run, source) { + const map = new Map(); + for (let i = 0; i < run.results.length; i++) { + const entry = run.results[i]; + const prefix = `${source}.results[${i}]`; + if (!isPlainObject(entry)) { + throw new Error(`${prefix} must be an object`); + } + const scenario = validateString(entry.scenario, `${prefix}.scenario`); + const framework = validateString(entry.framework, `${prefix}.framework`); + const params = normalizeParams(entry.params ?? {}, `${prefix}.params`); + const key = makeDescriptorKey(scenario, framework, params); + if (map.has(key)) { + throw new Error(`${source}: duplicate benchmark result key ${key}`); + } + map.set(key, entry); + } + return map; +} + +function getMetricValue(metrics, metricPath) { + const parts = metricPath.split("."); + let cursor = metrics; + for (const part of parts) { + if (!isPlainObject(cursor) || !(part in cursor)) { + return null; + } + cursor = cursor[part]; + } + if (typeof cursor !== "number" || !Number.isFinite(cursor)) { + return null; + } + return cursor; +} + +function overrideMatches(override, context) { + const { match } = override; + if (match.metric !== context.metric) return false; + if (match.scenario !== undefined && match.scenario !== context.scenario) return false; + if (match.framework !== undefined && match.framework !== context.framework) return false; + if (match.params !== undefined) { + if (stableStringify(match.params) !== stableStringify(context.params)) return false; + } + return true; +} + +function resolveRule(config, context) { + const base = config.defaultRules.get(context.metric); + if (!base) return null; + + let resolved = { + metric: base.metric, + relativeRegression: base.relativeRegression, + absoluteRegression: base.absoluteRegression, + severity: base.severity, + direction: base.direction, + }; + let source = base.source; + let reason = null; + + for (const override of config.overrides) { + if (!overrideMatches(override, context)) continue; + resolved = { + metric: resolved.metric, + relativeRegression: override.rulePatch.relativeRegression ?? resolved.relativeRegression, + absoluteRegression: override.rulePatch.absoluteRegression ?? resolved.absoluteRegression, + severity: override.rulePatch.severity ?? resolved.severity, + direction: override.rulePatch.direction ?? resolved.direction, + }; + source = override.source; + reason = override.reason; + } + + return { rule: resolved, source, reason }; +} + +function computeDeltaStats(baselineValue, currentValue, rule) { + const delta = currentValue - baselineValue; + const deltaPct = baselineValue === 0 ? null : delta / baselineValue; + const relativeAllowance = Math.abs(baselineValue) * rule.relativeRegression; + const allowedDelta = Math.max(relativeAllowance, rule.absoluteRegression); + const worseDelta = + rule.direction === "higher_is_worse" + ? currentValue - baselineValue + : baselineValue - currentValue; + const regressed = worseDelta > allowedDelta; + return { + delta, + deltaPct, + relativeAllowance, + allowedDelta, + worseDelta, + regressed, + }; +} + +function buildMissingResultCheck(req, metric, status, reason) { + return { + scenario: req.scenario, + framework: req.framework, + params: req.params, + metric, + status, + severity: "hard", + baselineValue: null, + currentValue: null, + delta: null, + deltaPct: null, + threshold: null, + ruleSource: null, + ruleReason: null, + reason, + }; +} + +function sortChecks(checks) { + checks.sort((a, b) => { + if (a.scenario !== b.scenario) return a.scenario.localeCompare(b.scenario); + const aParams = stableStringify(a.params ?? {}); + const bParams = stableStringify(b.params ?? {}); + if (aParams !== bParams) return aParams.localeCompare(bParams); + if (a.framework !== b.framework) return a.framework.localeCompare(b.framework); + return a.metric.localeCompare(b.metric); + }); + return checks; +} + +export function compareBenchRuns(baselineRunRaw, currentRunRaw, config, options = {}) { + const baselineRun = normalizeBenchRun(baselineRunRaw, "baseline"); + const currentRun = normalizeBenchRun(currentRunRaw, "current"); + const baselineIndex = indexBenchResults(baselineRun, "baseline"); + const currentIndex = indexBenchResults(currentRun, "current"); + + const metricPaths = [...config.defaultRules.keys()].sort(); + const checks = []; + + for (const req of config.requiredScenarios) { + const baselineResult = baselineIndex.get(req.key) ?? null; + const currentResult = currentIndex.get(req.key) ?? null; + + if (!baselineResult) { + for (const metric of metricPaths) { + checks.push( + buildMissingResultCheck( + req, + metric, + STATUS.MISSING_BASELINE_RESULT, + "required scenario missing from baseline results", + ), + ); + } + continue; + } + if (!currentResult) { + for (const metric of metricPaths) { + checks.push( + buildMissingResultCheck( + req, + metric, + STATUS.MISSING_CURRENT_RESULT, + "required scenario missing from current results", + ), + ); + } + continue; + } + + for (const metric of metricPaths) { + const resolved = resolveRule(config, { + scenario: req.scenario, + framework: req.framework, + params: req.params, + metric, + }); + if (!resolved) continue; + + const baselineValue = getMetricValue(baselineResult.metrics, metric); + const currentValue = getMetricValue(currentResult.metrics, metric); + + if (baselineValue === null) { + checks.push({ + scenario: req.scenario, + framework: req.framework, + params: req.params, + metric, + status: STATUS.MISSING_BASELINE_METRIC, + severity: "hard", + baselineValue: null, + currentValue, + delta: null, + deltaPct: null, + threshold: null, + ruleSource: resolved.source, + ruleReason: resolved.reason, + reason: `metric "${metric}" missing or non-numeric in baseline`, + }); + continue; + } + + if (currentValue === null) { + checks.push({ + scenario: req.scenario, + framework: req.framework, + params: req.params, + metric, + status: STATUS.MISSING_CURRENT_METRIC, + severity: "hard", + baselineValue, + currentValue: null, + delta: null, + deltaPct: null, + threshold: null, + ruleSource: resolved.source, + ruleReason: resolved.reason, + reason: `metric "${metric}" missing or non-numeric in current`, + }); + continue; + } + + const stats = computeDeltaStats(baselineValue, currentValue, resolved.rule); + const severity = resolved.rule.severity; + let status = STATUS.PASS; + if (stats.regressed) { + status = severity === "hard" ? STATUS.HARD_REGRESSION : STATUS.ADVISORY_REGRESSION; + } + + checks.push({ + scenario: req.scenario, + framework: req.framework, + params: req.params, + metric, + status, + severity, + baselineValue, + currentValue, + delta: stats.delta, + deltaPct: stats.deltaPct, + threshold: { + relativeRegression: resolved.rule.relativeRegression, + absoluteRegression: resolved.rule.absoluteRegression, + relativeAllowance: stats.relativeAllowance, + allowedDelta: stats.allowedDelta, + direction: resolved.rule.direction, + }, + ruleSource: resolved.source, + ruleReason: resolved.reason, + reason: stats.regressed ? "regression threshold exceeded" : "within threshold", + }); + } + } + + sortChecks(checks); + + const hardFailures = checks.filter((c) => HARD_STATUSES.has(c.status)); + const hardRegressions = checks.filter((c) => c.status === STATUS.HARD_REGRESSION); + const advisoryRegressions = checks.filter((c) => c.status === STATUS.ADVISORY_REGRESSION); + const missingChecks = checks.filter( + (c) => + c.status === STATUS.MISSING_BASELINE_RESULT || + c.status === STATUS.MISSING_CURRENT_RESULT || + c.status === STATUS.MISSING_BASELINE_METRIC || + c.status === STATUS.MISSING_CURRENT_METRIC, + ); + const passChecks = checks.filter((c) => c.status === STATUS.PASS); + + const failOnAdvisory = options.failOnAdvisory === true; + const exitCode = + hardFailures.length > 0 || (failOnAdvisory && advisoryRegressions.length > 0) ? 1 : 0; + + return { + meta: { + generatedAt: new Date().toISOString(), + failOnAdvisory, + gateBehavior: + "hard regressions fail CI; advisory regressions are informational unless --fail-on-advisory is set", + }, + summary: { + requiredScenarios: config.requiredScenarios.length, + metricsPerScenario: metricPaths.length, + totalChecks: checks.length, + passed: passChecks.length, + hardRegressions: hardRegressions.length, + advisoryRegressions: advisoryRegressions.length, + hardFailures: hardFailures.length, + missingComparisons: missingChecks.length, + exitCode, + }, + checks, + }; +} + +function formatNumber(value) { + if (value === null || value === undefined) return "n/a"; + return Number(value).toFixed(6); +} + +function formatPercent(value) { + if (value === null || value === undefined || !Number.isFinite(value)) return "n/a"; + const pct = value * 100; + const sign = pct >= 0 ? "+" : ""; + return `${sign}${pct.toFixed(2)}%`; +} + +function formatDelta(delta, deltaPct) { + if (delta === null || delta === undefined) return "n/a"; + const sign = delta >= 0 ? "+" : ""; + const pct = formatPercent(deltaPct); + return `${sign}${Number(delta).toFixed(6)} (${pct})`; +} + +function formatThreshold(threshold) { + if (!threshold) return "n/a"; + const dir = threshold.direction === "higher_is_worse" ? "+" : "-"; + return `${dir}${formatNumber(threshold.allowedDelta)} (max rel ${( + threshold.relativeRegression * 100 + ).toFixed(1)}%, abs ${formatNumber(threshold.absoluteRegression)})`; +} + +function paramsToString(params) { + const keys = Object.keys(params ?? {}).sort(); + if (keys.length === 0) return ""; + return keys.map((k) => `${k}=${String(params[k])}`).join(", "); +} + +function scenarioToString(check) { + const paramText = paramsToString(check.params); + if (paramText.length === 0) return `${check.scenario} [${check.framework}]`; + return `${check.scenario} (${paramText}) [${check.framework}]`; +} + +function statusToLabel(status) { + switch (status) { + case STATUS.PASS: + return "PASS"; + case STATUS.HARD_REGRESSION: + return "FAIL (hard)"; + case STATUS.ADVISORY_REGRESSION: + return "WARN (advisory)"; + case STATUS.MISSING_BASELINE_RESULT: + return "FAIL (missing baseline scenario)"; + case STATUS.MISSING_CURRENT_RESULT: + return "FAIL (missing current scenario)"; + case STATUS.MISSING_BASELINE_METRIC: + return "FAIL (missing baseline metric)"; + case STATUS.MISSING_CURRENT_METRIC: + return "FAIL (missing current metric)"; + default: + return status; + } +} + +function markdownTable(checks) { + if (checks.length === 0) { + return "_None._\n"; + } + + const lines = [ + "| Status | Scenario | Metric | Baseline | Current | Delta | Threshold |", + "| --- | --- | --- | ---: | ---: | ---: | --- |", + ]; + + for (const check of checks) { + lines.push( + [ + `| ${statusToLabel(check.status)}`, + `${scenarioToString(check)}`, + `${check.metric}`, + `${formatNumber(check.baselineValue)}`, + `${formatNumber(check.currentValue)}`, + `${formatDelta(check.delta, check.deltaPct)}`, + `${formatThreshold(check.threshold)} |`, + ].join(" | "), + ); + } + + return `${lines.join("\n")}\n`; +} + +export function toMarkdownReport(report, paths) { + const hard = report.checks.filter((c) => HARD_STATUSES.has(c.status)); + const advisory = report.checks.filter((c) => c.status === STATUS.ADVISORY_REGRESSION); + const allChecks = report.checks; + + const lines = [ + "# CI Benchmark Regression Report", + "", + `- Generated: ${report.meta.generatedAt}`, + `- Baseline: \`${toRel(paths.baselinePath)}\``, + `- Current: \`${toRel(paths.currentPath)}\``, + `- Config: \`${toRel(paths.configPath)}\``, + `- Gate behavior: ${report.meta.gateBehavior}`, + "", + "## Summary", + "", + "| Checks | Value |", + "| --- | ---: |", + `| Required scenarios | ${report.summary.requiredScenarios} |`, + `| Metrics per scenario | ${report.summary.metricsPerScenario} |`, + `| Total checks | ${report.summary.totalChecks} |`, + `| Passed | ${report.summary.passed} |`, + `| Hard regressions | ${report.summary.hardRegressions} |`, + `| Advisory regressions | ${report.summary.advisoryRegressions} |`, + `| Missing comparisons | ${report.summary.missingComparisons} |`, + `| Exit code | ${report.summary.exitCode} |`, + "", + "## Hard Failures", + "", + markdownTable(hard), + "## Advisory Regressions", + "", + markdownTable(advisory), + "## All Checks", + "", + markdownTable(allChecks), + ]; + + return `${lines.join("\n")}\n`; +} + +function toRel(path) { + if (path.startsWith(ROOT)) { + return relative(ROOT, path); + } + return path; +} + +function writeOutput(path, content) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, "utf8"); +} + +function printConsoleSummary(report, paths) { + const hard = report.checks.filter((c) => HARD_STATUSES.has(c.status)); + const advisory = report.checks.filter((c) => c.status === STATUS.ADVISORY_REGRESSION); + + process.stdout.write( + [ + `bench-ci-compare: checked ${report.summary.totalChecks} comparisons`, + ` hard regressions: ${report.summary.hardRegressions}`, + ` advisory regressions: ${report.summary.advisoryRegressions}`, + ` missing comparisons: ${report.summary.missingComparisons}`, + ` report json: ${toRel(paths.reportJsonPath)}`, + ` report md: ${toRel(paths.reportMdPath)}`, + "", + ].join("\n"), + ); + + if (hard.length > 0) { + process.stderr.write("bench-ci-compare: hard failures detected\n"); + for (const check of hard) { + process.stderr.write( + [ + `- scenario=${scenarioToString(check)}`, + `metric=${check.metric}`, + `baseline=${formatNumber(check.baselineValue)}`, + `current=${formatNumber(check.currentValue)}`, + `delta=${formatDelta(check.delta, check.deltaPct)}`, + `threshold=${formatThreshold(check.threshold)}`, + `reason=${check.reason}`, + ].join(" | "), + ); + process.stderr.write("\n"); + } + } + + if (hard.length === 0 && advisory.length > 0) { + process.stdout.write("bench-ci-compare: advisory regressions (non-fatal)\n"); + for (const check of advisory) { + process.stdout.write( + [ + `- scenario=${scenarioToString(check)}`, + `metric=${check.metric}`, + `baseline=${formatNumber(check.baselineValue)}`, + `current=${formatNumber(check.currentValue)}`, + `delta=${formatDelta(check.delta, check.deltaPct)}`, + `threshold=${formatThreshold(check.threshold)}`, + ].join(" | "), + ); + process.stdout.write("\n"); + } + } +} + +function runCli() { + const opts = parseArgs(process.argv); + + const baselineRunRaw = readJson(opts.baselinePath, "baseline"); + const currentRunRaw = readJson(opts.currentPath, "current"); + const configRaw = readJson(opts.configPath, "config"); + const config = normalizeComparatorConfig(configRaw); + + const report = compareBenchRuns(baselineRunRaw, currentRunRaw, config, { + failOnAdvisory: opts.failOnAdvisory, + }); + + const reportWithPaths = { + ...report, + inputs: { + baselinePath: opts.baselinePath, + currentPath: opts.currentPath, + configPath: opts.configPath, + }, + baselineMeta: isPlainObject(baselineRunRaw.meta) ? baselineRunRaw.meta : null, + currentMeta: isPlainObject(currentRunRaw.meta) ? currentRunRaw.meta : null, + configMeta: config.meta, + }; + + writeOutput(`${opts.reportJsonPath}`, `${JSON.stringify(reportWithPaths, null, 2)}\n`); + writeOutput(opts.reportMdPath, toMarkdownReport(report, opts)); + printConsoleSummary(report, opts); + process.exit(report.summary.exitCode); +} + +const invokedPath = process.argv[1] ? realpathSync(process.argv[1]) : null; +const selfPath = realpathSync(fileURLToPath(import.meta.url)); + +if (invokedPath && invokedPath === selfPath) { + try { + runCli(); + } catch (err) { + const detail = err instanceof Error ? (err.stack ?? err.message) : String(err); + process.stderr.write(`bench-ci-compare: FAIL\n${detail}\n`); + process.exit(1); + } +} diff --git a/scripts/run-bench-ci.mjs b/scripts/run-bench-ci.mjs new file mode 100644 index 00000000..a9a52aef --- /dev/null +++ b/scripts/run-bench-ci.mjs @@ -0,0 +1,379 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = fileURLToPath(new URL("..", import.meta.url)); +const PROFILE_PATH = join(ROOT, "packages", "bench", "profiles", "ci.json"); +const BENCH_RUNNER_PATH = join(ROOT, "packages", "bench", "dist", "run.js"); +const DEFAULT_JSON_OUTPUT = "results.json"; +const DEFAULT_MARKDOWN_OUTPUT = "results.md"; + +function fail(message) { + process.stderr.write(`run-bench-ci: ${message}\n`); + process.exit(1); +} + +function printHelp() { + process.stdout.write( + [ + "Usage:", + " node scripts/run-bench-ci.mjs [--output-dir ]", + "", + "Options:", + " --output-dir Override profile output directory", + " -h, --help Show this help", + "", + ].join("\n"), + ); +} + +function parseArgs(argv) { + let outputDir = null; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + + if (arg === "--output-dir") { + outputDir = argv[++i] ?? null; + if (!outputDir) fail("missing value for --output-dir"); + continue; + } + + if (arg === "--help" || arg === "-h") { + printHelp(); + process.exit(0); + } + + fail(`unknown argument: ${arg}`); + } + + return { outputDir }; +} + +function toAbsPath(pathValue) { + return isAbsolute(pathValue) ? pathValue : resolve(ROOT, pathValue); +} + +function readJson(path, label) { + let raw = ""; + try { + raw = readFileSync(path, "utf8"); + } catch (error) { + fail( + `${label}: failed to read ${path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch (error) { + fail( + `${label}: failed to parse ${path}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return parsed; +} + +function isPlainObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeParams(value, source) { + if (!isPlainObject(value)) { + fail(`${source} must be an object`); + } + const out = {}; + const keys = Object.keys(value).sort(); + for (const key of keys) { + const raw = value[key]; + if (typeof raw !== "number" && typeof raw !== "string" && typeof raw !== "boolean") { + fail( + `${source}.${key} must be number|string|boolean (got ${raw === null ? "null" : typeof raw})`, + ); + } + out[key] = raw; + } + return out; +} + +function stableStringify(value) { + if (Array.isArray(value)) { + return `[${value.map((entry) => stableStringify(entry)).join(",")}]`; + } + if (isPlainObject(value)) { + const keys = Object.keys(value).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function makeKey(scenario, framework, params) { + return `${scenario}|${framework}|${stableStringify(params ?? {})}`; +} + +function loadCiProfile(profilePath) { + if (!existsSync(profilePath)) { + fail(`profile file not found: ${profilePath}`); + } + + const profile = readJson(profilePath, "profile"); + if (!isPlainObject(profile)) { + fail("profile root must be an object"); + } + + if (profile.name !== "ci") { + fail('profile.name must be "ci"'); + } + if (profile.suite !== "terminal") { + fail('profile.suite must be "terminal"'); + } + if (profile.io !== "stub") { + fail('profile.io must be "stub" for deterministic PR CI gating'); + } + + if (typeof profile.framework !== "string" || profile.framework.length === 0) { + fail("profile.framework must be a non-empty string"); + } + if (!Number.isInteger(profile.warmup) || profile.warmup < 0) { + fail("profile.warmup must be an integer >= 0"); + } + if (!Number.isInteger(profile.iterations) || profile.iterations <= 0) { + fail("profile.iterations must be an integer > 0"); + } + if (typeof profile.outputDir !== "string" || profile.outputDir.length === 0) { + fail("profile.outputDir must be a non-empty string"); + } + + if (!Array.isArray(profile.requiredScenarios) || profile.requiredScenarios.length === 0) { + fail("profile.requiredScenarios must be a non-empty array"); + } + const requiredScenarios = []; + const seenScenarios = new Set(); + for (const scenario of profile.requiredScenarios) { + if (typeof scenario !== "string" || scenario.length === 0) { + fail("profile.requiredScenarios entries must be non-empty strings"); + } + if (!scenario.startsWith("terminal-")) { + fail(`profile.requiredScenarios entry must be a terminal scenario: ${scenario}`); + } + if (seenScenarios.has(scenario)) { + fail(`profile.requiredScenarios contains duplicate entry: ${scenario}`); + } + seenScenarios.add(scenario); + requiredScenarios.push(scenario); + } + + if (!isPlainObject(profile.requiredScenarioParams)) { + fail("profile.requiredScenarioParams must be an object"); + } + const requiredScenarioParams = new Map(); + for (const [scenario, rawParamSets] of Object.entries(profile.requiredScenarioParams)) { + if (!Array.isArray(rawParamSets) || rawParamSets.length === 0) { + fail(`profile.requiredScenarioParams.${scenario} must be a non-empty array`); + } + if (!seenScenarios.has(scenario)) { + fail(`profile.requiredScenarioParams.${scenario} is not listed in requiredScenarios`); + } + const normalized = rawParamSets.map((params, index) => + normalizeParams(params, `profile.requiredScenarioParams.${scenario}[${index}]`), + ); + requiredScenarioParams.set(scenario, normalized); + } + + const outputBlock = isPlainObject(profile.output) ? profile.output : {}; + const resultsJson = + typeof outputBlock.resultsJson === "string" && outputBlock.resultsJson.length > 0 + ? outputBlock.resultsJson + : DEFAULT_JSON_OUTPUT; + const resultsMarkdown = + typeof outputBlock.resultsMarkdown === "string" && outputBlock.resultsMarkdown.length > 0 + ? outputBlock.resultsMarkdown + : DEFAULT_MARKDOWN_OUTPUT; + + if ( + !isPlainObject(profile.deterministicOrdering) || + Object.keys(profile.deterministicOrdering).length === 0 + ) { + fail("profile.deterministicOrdering must be an object"); + } + + return { + name: profile.name, + description: typeof profile.description === "string" ? profile.description : "", + suite: profile.suite, + framework: profile.framework, + io: profile.io, + warmup: profile.warmup, + iterations: profile.iterations, + outputDir: profile.outputDir, + output: { resultsJson, resultsMarkdown }, + requiredScenarios, + requiredScenarioParams, + deterministicOrdering: profile.deterministicOrdering, + }; +} + +function runBench(profile, outputDir) { + const args = [ + "--expose-gc", + BENCH_RUNNER_PATH, + "--suite", + profile.suite, + "--framework", + profile.framework, + "--warmup", + String(profile.warmup), + "--iterations", + String(profile.iterations), + "--io", + profile.io, + "--output-dir", + outputDir, + ]; + + process.stdout.write("\n[bench:ci] running reduced CI suite\n"); + process.stdout.write(` suite=${profile.suite}\n`); + process.stdout.write(` framework=${profile.framework}\n`); + process.stdout.write(` io=${profile.io}\n`); + process.stdout.write(` warmup=${profile.warmup}\n`); + process.stdout.write(` iterations=${profile.iterations}\n`); + + const result = spawnSync(process.execPath, args, { + cwd: ROOT, + stdio: "inherit", + env: { ...process.env }, + }); + + if (result.signal) { + fail(`benchmark command terminated by signal ${result.signal}`); + } + + if ((result.status ?? 1) !== 0) { + process.exit(result.status ?? 1); + } +} + +function loadBenchRun(path) { + const raw = readJson(path, "results"); + if (!isPlainObject(raw) || !Array.isArray(raw.results)) { + fail(`results: ${path} must be a benchmark run object with a results[] array`); + } + return raw; +} + +function verifyCoverage(profile, run) { + const index = new Map(); + + for (const entry of run.results) { + if (!isPlainObject(entry)) continue; + if (entry.framework !== profile.framework) continue; + const scenario = + typeof entry.scenario === "string" && entry.scenario.length > 0 ? entry.scenario : null; + if (!scenario) continue; + const params = normalizeParams(entry.params ?? {}, `results.params(${scenario})`); + const key = makeKey(scenario, profile.framework, params); + index.set(key, true); + } + + for (const scenario of profile.requiredScenarios) { + const hasAnyForScenario = [...index.keys()].some((key) => + key.startsWith(`${scenario}|${profile.framework}|`), + ); + if (!hasAnyForScenario) { + fail(`required scenario missing from output: ${scenario} [${profile.framework}]`); + } + } + + for (const [scenario, paramSets] of profile.requiredScenarioParams.entries()) { + for (const params of paramSets) { + const key = makeKey(scenario, profile.framework, params); + if (!index.has(key)) { + fail( + `required scenario+params missing from output: ${scenario} [${profile.framework}] params=${stableStringify( + params, + )}`, + ); + } + } + } + + return { + requiredScenarios: profile.requiredScenarios.length, + requiredParamRows: [...profile.requiredScenarioParams.values()].reduce( + (sum, paramSets) => sum + paramSets.length, + 0, + ), + frameworkResultRows: [...index.keys()].length, + }; +} + +function main() { + const { outputDir: outputDirArg } = parseArgs(process.argv.slice(2)); + const profile = loadCiProfile(PROFILE_PATH); + + if (!existsSync(BENCH_RUNNER_PATH)) { + fail(`missing bench runner: ${BENCH_RUNNER_PATH}. Run \"npm run build\" first.`); + } + + const outputDir = toAbsPath(outputDirArg ?? profile.outputDir); + mkdirSync(outputDir, { recursive: true }); + + runBench(profile, outputDir); + + const resultsJsonPath = join(outputDir, profile.output.resultsJson); + const resultsMarkdownPath = join(outputDir, profile.output.resultsMarkdown); + + if (!existsSync(resultsJsonPath)) { + fail(`bench output missing JSON report: ${resultsJsonPath}`); + } + if (!existsSync(resultsMarkdownPath)) { + fail(`bench output missing markdown report: ${resultsMarkdownPath}`); + } + + const run = loadBenchRun(resultsJsonPath); + const coverage = verifyCoverage(profile, run); + + const manifest = { + profileName: profile.name, + profilePath: PROFILE_PATH, + generatedAt: new Date().toISOString(), + outputDir, + invocation: { + suite: profile.suite, + framework: profile.framework, + io: profile.io, + warmup: profile.warmup, + iterations: profile.iterations, + }, + output: { + resultsJson: resultsJsonPath, + resultsMarkdown: resultsMarkdownPath, + }, + deterministicOrdering: profile.deterministicOrdering, + coverage, + }; + + writeFileSync(join(outputDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + writeFileSync( + join(outputDir, "profile.json"), + `${JSON.stringify( + { + ...profile, + requiredScenarioParams: Object.fromEntries(profile.requiredScenarioParams.entries()), + }, + null, + 2, + )}\n`, + "utf8", + ); + + process.stdout.write( + `\n[bench:ci] completed. Output: ${outputDir}\n[bench:ci] verified ${coverage.frameworkResultRows} result row(s)\n`, + ); +} + +main(); From 263dbe6eb8b2eb73d941f363c5069f1e30a4fee5 Mon Sep 17 00:00:00 2001 From: RtlZeroMemory <58250858+RtlZeroMemory@users.noreply.github.com> Date: Mon, 16 Feb 2026 12:09:28 +0400 Subject: [PATCH 2/3] Fix perf gate bench build on fresh CI runners --- .github/workflows/ci.yml | 3 +++ packages/bench/src/shims.d.ts | 1 + packages/bench/tsconfig.json | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf133a94..4a0acda5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,9 @@ jobs: - name: Build packages run: npm run build + - name: Build benchmark package + run: npx tsc -p packages/bench/tsconfig.json + - name: Run reduced benchmark suite run: npm run bench:ci -- --output-dir .artifacts/bench/ci diff --git a/packages/bench/src/shims.d.ts b/packages/bench/src/shims.d.ts index 5b213eb2..08495327 100644 --- a/packages/bench/src/shims.d.ts +++ b/packages/bench/src/shims.d.ts @@ -1 +1,2 @@ declare module "blessed"; +declare module "@rezi-ui/ink-compat"; diff --git a/packages/bench/tsconfig.json b/packages/bench/tsconfig.json index 4c3c6b37..fb8be15e 100644 --- a/packages/bench/tsconfig.json +++ b/packages/bench/tsconfig.json @@ -9,5 +9,5 @@ "jsxImportSource": "react" }, "include": ["src/**/*.ts", "src/**/*.tsx"], - "references": [{ "path": "../core" }, { "path": "../ink-compat" }] + "references": [{ "path": "../core" }] } From 69a6011722a547409b637093d7a82577efb952e1 Mon Sep 17 00:00:00 2001 From: RtlZeroMemory <58250858+RtlZeroMemory@users.noreply.github.com> Date: Mon, 16 Feb 2026 12:12:02 +0400 Subject: [PATCH 3/3] Reseed perf baseline from GitHub runner artifacts --- benchmarks/ci-baseline/config.json | 7 +- benchmarks/ci-baseline/results.json | 340 ++++++++++++++-------------- benchmarks/ci-baseline/results.md | 22 +- docs/dev/perf-regressions.md | 2 +- 4 files changed, 186 insertions(+), 185 deletions(-) diff --git a/benchmarks/ci-baseline/config.json b/benchmarks/ci-baseline/config.json index 18729d50..d8fc416c 100644 --- a/benchmarks/ci-baseline/config.json +++ b/benchmarks/ci-baseline/config.json @@ -1,10 +1,11 @@ { "meta": { "profilePath": "packages/bench/profiles/ci.json", - "baselinePolicy": "Update only on intentional perf shifts or deliberate threshold tuning.", - "baselineGenerationCommand": "npm run bench:ci -- --output-dir benchmarks/ci-baseline", + "baselinePolicy": "Update only on intentional perf shifts or deliberate threshold tuning. Prefer refreshing from CI perf-regression artifacts to stay host-class consistent.", + "baselineGenerationCommand": "Use perf-regression GitHub Actions artifacts (results.json/results.md) and copy into benchmarks/ci-baseline/", + "baselineSource": "GitHub Actions run 22054843644 / perf regression gate artifact", "comparisonCommandTemplate": "npm run bench:ci:compare -- --current /results.json --report-json /compare.json --report-md /compare.md", - "createdAt": "2026-02-16T06:42:55Z" + "createdAt": "2026-02-16T08:10:03Z" }, "requiredScenarios": [ { diff --git a/benchmarks/ci-baseline/results.json b/benchmarks/ci-baseline/results.json index 31987728..30de709f 100644 --- a/benchmarks/ci-baseline/results.json +++ b/benchmarks/ci-baseline/results.json @@ -1,14 +1,14 @@ { "meta": { - "timestamp": "2026-02-16T06:57:58.155Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:03.477Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64", "osType": "Linux", - "osRelease": "6.6.87.2-microsoft-standard-WSL2", - "cpuModel": "AMD Ryzen 7 9800X3D 8-Core Processor", - "cpuCores": 12, - "memoryTotalMb": 15993 + "osRelease": "6.14.0-1017-azure", + "cpuModel": "AMD EPYC 7763 64-Core Processor", + "cpuCores": 4, + "memoryTotalMb": 15990 }, "invocation": { "suite": "terminal", @@ -27,55 +27,55 @@ "metrics": { "timing": { "n": 800, - "mean": 0.025910159999999766, - "median": 0.016370999999992364, - "p95": 0.0790140000000008, - "p99": 0.11780600000000163, - "min": 0.00970999999998412, - "max": 0.42496099999999615, - "stddev": 0.034548499526925215, - "cv": 1.3333958388109346, - "meanCi95Low": 0.023927173750000003, - "meanCi95High": 0.028436486249999095 + "mean": 0.07360876750000024, + "median": 0.07055650000000924, + "p95": 0.1044949999999858, + "p99": 0.12695700000000443, + "min": 0.04966299999998114, + "max": 0.48179700000000025, + "stddev": 0.02356119323885035, + "cv": 0.320086778233996, + "meanCi95Low": 0.0722970862499998, + "meanCi95High": 0.07557613500000024 }, "memBefore": { - "rssKb": 60324, - "heapUsedKb": 8345, - "heapTotalKb": 17448, - "externalKb": 1816, - "arrayBuffersKb": 44 + "rssKb": 69036, + "heapUsedKb": 9570, + "heapTotalKb": 27300, + "externalKb": 1926, + "arrayBuffersKb": 52 }, "memAfter": { - "rssKb": 65532, - "heapUsedKb": 10767, - "heapTotalKb": 19240, - "externalKb": 1816, - "arrayBuffersKb": 44 + "rssKb": 77960, + "heapUsedKb": 15784, + "heapTotalKb": 27812, + "externalKb": 1926, + "arrayBuffersKb": 52 }, "memPeak": { - "rssKb": 65532, - "heapUsedKb": 13549, - "heapTotalKb": 19240, - "externalKb": 1816, - "arrayBuffersKb": 44 + "rssKb": 77960, + "heapUsedKb": 16619, + "heapTotalKb": 27812, + "externalKb": 1926, + "arrayBuffersKb": 52 }, - "rssGrowthKb": 5208, - "heapUsedGrowthKb": 2422, + "rssGrowthKb": 8924, + "heapUsedGrowthKb": 6214, "rssSlopeKbPerIter": null, "heapUsedSlopeKbPerIter": null, "memStable": null, "cpu": { - "userMs": 47.138000000000005, - "systemMs": 2.830000000000002 + "userMs": 80.51500000000001, + "systemMs": 5.827999999999999 }, "iterations": 800, - "totalWallMs": 21.124217000000016, - "opsPerSec": 37871.22618556699, + "totalWallMs": 59.90091700000002, + "opsPerSec": 13355.38819881505, "framesProduced": 800, "bytesProduced": 185652 }, - "timestamp": "2026-02-16T06:57:58.271Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:03.608Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64" }, @@ -90,55 +90,55 @@ "metrics": { "timing": { "n": 800, - "mean": 0.04680949749999994, - "median": 0.03033600000000547, - "p95": 0.11275600000000452, - "p99": 0.30505500000001007, - "min": 0.02400200000002428, - "max": 0.48030400000001805, - "stddev": 0.04769691433552194, - "cv": 1.0189580508853358, - "meanCi95Low": 0.04411405750000012, - "meanCi95High": 0.049964247500000204 + "mean": 0.1549332012499995, + "median": 0.13179999999999836, + "p95": 0.3043260000000032, + "p99": 0.3960480000000075, + "min": 0.08166199999999435, + "max": 0.5906970000000058, + "stddev": 0.07791041018534198, + "cv": 0.5028645219795471, + "meanCi95Low": 0.15010021124999887, + "meanCi95High": 0.16005556249999997 }, "memBefore": { - "rssKb": 64416, - "heapUsedKb": 8507, - "heapTotalKb": 17960, - "externalKb": 1828, - "arrayBuffersKb": 57 + "rssKb": 69456, + "heapUsedKb": 9719, + "heapTotalKb": 28068, + "externalKb": 1938, + "arrayBuffersKb": 63 }, "memAfter": { - "rssKb": 77060, - "heapUsedKb": 25188, - "heapTotalKb": 30248, - "externalKb": 1836, + "rssKb": 92244, + "heapUsedKb": 22275, + "heapTotalKb": 39844, + "externalKb": 1937, "arrayBuffersKb": 64 }, "memPeak": { - "rssKb": 77060, - "heapUsedKb": 25188, - "heapTotalKb": 30248, - "externalKb": 1836, - "arrayBuffersKb": 64 + "rssKb": 92244, + "heapUsedKb": 24848, + "heapTotalKb": 39844, + "externalKb": 1952, + "arrayBuffersKb": 79 }, - "rssGrowthKb": 12644, - "heapUsedGrowthKb": 16681, + "rssGrowthKb": 22788, + "heapUsedGrowthKb": 12556, "rssSlopeKbPerIter": null, "heapUsedSlopeKbPerIter": null, "memStable": null, "cpu": { - "userMs": 62.85900000000001, - "systemMs": 14.193000000000005 + "userMs": 192.20499999999998, + "systemMs": 17.858 }, "iterations": 800, - "totalWallMs": 37.875388, - "opsPerSec": 21121.89583377997, + "totalWallMs": 125.00502299999997, + "opsPerSec": 6399.742832734011, "framesProduced": 800, "bytesProduced": 275200 }, - "timestamp": "2026-02-16T06:57:58.417Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:03.845Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64" }, @@ -153,55 +153,55 @@ "metrics": { "timing": { "n": 800, - "mean": 0.11282477749999989, - "median": 0.09059450000000879, - "p95": 0.2054510000000107, - "p99": 0.45930300000000557, - "min": 0.06829400000000874, - "max": 0.9811590000000194, - "stddev": 0.07118812704328271, - "cv": 0.630961820804679, - "meanCi95Low": 0.10867026499999985, - "meanCi95High": 0.11813537499999936 + "mean": 0.2646597137499987, + "median": 0.2055080000000089, + "p95": 0.6717710000000068, + "p99": 0.8909590000000094, + "min": 0.17999500000001944, + "max": 1.1083440000000166, + "stddev": 0.15271748812435376, + "cv": 0.5770333760302214, + "meanCi95Low": 0.2553964587499987, + "meanCi95High": 0.2754517275000001 }, "memBefore": { - "rssKb": 64304, - "heapUsedKb": 8864, - "heapTotalKb": 17960, - "externalKb": 1939, - "arrayBuffersKb": 168 + "rssKb": 71596, + "heapUsedKb": 9744, + "heapTotalKb": 28068, + "externalKb": 2049, + "arrayBuffersKb": 86 }, "memAfter": { - "rssKb": 81376, - "heapUsedKb": 14866, - "heapTotalKb": 32552, - "externalKb": 1904, - "arrayBuffersKb": 133 + "rssKb": 99652, + "heapUsedKb": 31420, + "heapTotalKb": 46244, + "externalKb": 2058, + "arrayBuffersKb": 185 }, "memPeak": { - "rssKb": 87648, - "heapUsedKb": 26593, - "heapTotalKb": 39976, - "externalKb": 2235, - "arrayBuffersKb": 463 + "rssKb": 99652, + "heapUsedKb": 31420, + "heapTotalKb": 46244, + "externalKb": 2246, + "arrayBuffersKb": 373 }, - "rssGrowthKb": 17072, - "heapUsedGrowthKb": 6002, + "rssGrowthKb": 28056, + "heapUsedGrowthKb": 21676, "rssSlopeKbPerIter": null, "heapUsedSlopeKbPerIter": null, "memStable": null, "cpu": { - "userMs": 131.20600000000002, - "systemMs": 17.587999999999997 + "userMs": 318.20799999999997, + "systemMs": 34.279 }, "iterations": 800, - "totalWallMs": 90.722006, - "opsPerSec": 8818.147164867585, + "totalWallMs": 213.08038200000001, + "opsPerSec": 3754.451688565116, "framesProduced": 800, "bytesProduced": 5747200 }, - "timestamp": "2026-02-16T06:57:58.585Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:04.161Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64" }, @@ -215,55 +215,55 @@ "metrics": { "timing": { "n": 800, - "mean": 0.26352908125000024, - "median": 0.22015100000001553, - "p95": 0.6459629999999947, - "p99": 0.9262669999999957, - "min": 0.17140899999998283, - "max": 1.5334569999999985, - "stddev": 0.15578726030507803, - "cv": 0.5911577559718673, - "meanCi95Low": 0.2545057762499992, - "meanCi95High": 0.27491137999999965 + "mean": 0.634011616250002, + "median": 0.5162965000000099, + "p95": 1.5324840000000108, + "p99": 2.0287379999999757, + "min": 0.42571300000008705, + "max": 2.8421220000000176, + "stddev": 0.3608315920056282, + "cv": 0.5691245755714134, + "meanCi95Low": 0.6121057937500001, + "meanCi95High": 0.6599321362500022 }, "memBefore": { - "rssKb": 83656, - "heapUsedKb": 9458, - "heapTotalKb": 26664, - "externalKb": 1820, - "arrayBuffersKb": 48 + "rssKb": 81516, + "heapUsedKb": 10501, + "heapTotalKb": 28068, + "externalKb": 1930, + "arrayBuffersKb": 56 }, "memAfter": { - "rssKb": 128284, - "heapUsedKb": 29371, - "heapTotalKb": 79656, - "externalKb": 1820, - "arrayBuffersKb": 49 + "rssKb": 154428, + "heapUsedKb": 39734, + "heapTotalKb": 74916, + "externalKb": 1930, + "arrayBuffersKb": 57 }, "memPeak": { - "rssKb": 136332, - "heapUsedKb": 69042, - "heapTotalKb": 88104, - "externalKb": 1820, - "arrayBuffersKb": 49 + "rssKb": 154428, + "heapUsedKb": 72075, + "heapTotalKb": 95140, + "externalKb": 1930, + "arrayBuffersKb": 57 }, - "rssGrowthKb": 44628, - "heapUsedGrowthKb": 19913, + "rssGrowthKb": 72912, + "heapUsedGrowthKb": 29233, "rssSlopeKbPerIter": null, "heapUsedSlopeKbPerIter": null, "memStable": null, "cpu": { - "userMs": 267.14699999999993, - "systemMs": 29.964000000000002 + "userMs": 688.113, + "systemMs": 69.674 }, "iterations": 800, - "totalWallMs": 211.489891, - "opsPerSec": 3782.6867100706954, + "totalWallMs": 508.560597, + "opsPerSec": 1573.0672111036554, "framesProduced": 800, "bytesProduced": 3915584 }, - "timestamp": "2026-02-16T06:57:58.822Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:04.576Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64" }, @@ -277,55 +277,55 @@ "metrics": { "timing": { "n": 800, - "mean": 0.0668960075000001, - "median": 0.04199199999999337, - "p95": 0.1597680000000139, - "p99": 0.4018700000000024, - "min": 0.029821999999995796, - "max": 0.9898000000000025, - "stddev": 0.07284240508409222, - "cv": 1.0888901715710328, - "meanCi95Low": 0.06280069249999957, - "meanCi95High": 0.07206160375000088 + "mean": 0.1941018637499995, + "median": 0.16234249999999406, + "p95": 0.4190100000000143, + "p99": 0.4937900000000468, + "min": 0.09962600000000066, + "max": 0.8208989999999972, + "stddev": 0.09698325279806252, + "cv": 0.4996513218594109, + "meanCi95Low": 0.18804177374999997, + "meanCi95High": 0.2005880249999991 }, "memBefore": { - "rssKb": 63776, - "heapUsedKb": 8735, - "heapTotalKb": 17704, - "externalKb": 1830, - "arrayBuffersKb": 58 + "rssKb": 71980, + "heapUsedKb": 9827, + "heapTotalKb": 28324, + "externalKb": 1939, + "arrayBuffersKb": 66 }, "memAfter": { - "rssKb": 77956, - "heapUsedKb": 15193, - "heapTotalKb": 35624, - "externalKb": 1889, - "arrayBuffersKb": 118 + "rssKb": 93076, + "heapUsedKb": 25024, + "heapTotalKb": 40100, + "externalKb": 1999, + "arrayBuffersKb": 126 }, "memPeak": { - "rssKb": 77956, - "heapUsedKb": 23596, - "heapTotalKb": 37416, - "externalKb": 1889, - "arrayBuffersKb": 118 + "rssKb": 93076, + "heapUsedKb": 28553, + "heapTotalKb": 40100, + "externalKb": 1999, + "arrayBuffersKb": 126 }, - "rssGrowthKb": 14180, - "heapUsedGrowthKb": 6458, + "rssGrowthKb": 21096, + "heapUsedGrowthKb": 15197, "rssSlopeKbPerIter": null, "heapUsedSlopeKbPerIter": null, "memStable": null, "cpu": { - "userMs": 107.134, - "systemMs": 12.023000000000003 + "userMs": 260.919, + "systemMs": 20.800999999999995 }, "iterations": 800, - "totalWallMs": 53.926028, - "opsPerSec": 14835.136754370264, + "totalWallMs": 156.412731, + "opsPerSec": 5114.673178361677, "framesProduced": 800, "bytesProduced": 660640 }, - "timestamp": "2026-02-16T06:57:59.221Z", - "nodeVersion": "v20.19.5", + "timestamp": "2026-02-16T08:10:05.372Z", + "nodeVersion": "v22.22.0", "platform": "linux", "arch": "x64" } diff --git a/benchmarks/ci-baseline/results.md b/benchmarks/ci-baseline/results.md index efb6917e..a78534f6 100644 --- a/benchmarks/ci-baseline/results.md +++ b/benchmarks/ci-baseline/results.md @@ -1,6 +1,6 @@ # Benchmark Results -> 2026-02-16T06:57:58.155Z | Node v20.19.5 | Linux 6.6.87.2-microsoft-standard-WSL2 | linux x64 | AMD Ryzen 7 9800X3D 8-Core Processor (12 cores) | RAM 15993MB +> 2026-02-16T08:10:03.477Z | Node v22.22.0 | Linux 6.14.0-1017-azure | linux x64 | AMD EPYC 7763 64-Core Processor (4 cores) | RAM 15990MB > Invocation: suite=terminal scenario=all framework=rezi-native warmup=50 iterations=800 quick=no io=stub @@ -8,31 +8,31 @@ | Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Rezi (native) | 26µs | 35µs | 24µs–28µs | 37.9K ops/s | 21.12ms | 47.14ms | 2.83ms | 64.0MB | 13.2MB | 181.30078125KB | +| Rezi (native) | 74µs | 24µs | 72µs–76µs | 13.4K ops/s | 59.90ms | 80.52ms | 5.83ms | 76.1MB | 16.2MB | 181.30078125KB | ## terminal-frame-fill (rows=40, cols=120, dirtyLines=1) | Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Rezi (native) | 47µs | 48µs | 44µs–50µs | 21.1K ops/s | 37.88ms | 62.86ms | 14.19ms | 75.3MB | 24.6MB | 268.75KB | +| Rezi (native) | 155µs | 78µs | 150µs–160µs | 6.4K ops/s | 125.01ms | 192.20ms | 17.86ms | 90.1MB | 24.3MB | 268.75KB | ## terminal-frame-fill (rows=40, cols=120, dirtyLines=40) | Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Rezi (native) | 113µs | 71µs | 109µs–118µs | 8.8K ops/s | 90.72ms | 131.21ms | 17.59ms | 85.6MB | 26.0MB | 5.5MB | +| Rezi (native) | 265µs | 153µs | 255µs–275µs | 3.8K ops/s | 213.08ms | 318.21ms | 34.28ms | 97.3MB | 30.7MB | 5.5MB | ## terminal-virtual-list (items=100000, viewport=40) | Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Rezi (native) | 264µs | 156µs | 255µs–275µs | 3.8K ops/s | 211.49ms | 267.15ms | 29.96ms | 133.1MB | 67.4MB | 3.7MB | +| Rezi (native) | 634µs | 361µs | 612µs–660µs | 1.6K ops/s | 508.56ms | 688.11ms | 69.67ms | 150.8MB | 70.4MB | 3.7MB | ## terminal-table (rows=40, cols=8) | Framework | Mean | Std dev | Mean CI95 | ops/s | Wall | CPU user | CPU sys | Peak RSS | Peak Heap | Bytes | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| Rezi (native) | 67µs | 73µs | 63µs–72µs | 14.8K ops/s | 53.93ms | 107.13ms | 12.02ms | 76.1MB | 23.0MB | 645.15625KB | +| Rezi (native) | 194µs | 97µs | 188µs–201µs | 5.1K ops/s | 156.41ms | 260.92ms | 20.80ms | 90.9MB | 27.9MB | 645.15625KB | ## Relative Performance (vs Rezi native) @@ -50,8 +50,8 @@ | Scenario | Framework | Peak RSS | Peak Heap | RSS Growth | Heap Growth | RSS Slope | Stable | |---|---|---:|---:|---:|---:|---:|---:| -| terminal-rerender | Rezi (native) | 64.0MB | 13.2MB | +5.1MB | +2.4MB | N/A | N/A | -| terminal-frame-fill (rows=40, cols=120, dirtyLines=1) | Rezi (native) | 75.3MB | 24.6MB | +12.3MB | +16.3MB | N/A | N/A | -| terminal-frame-fill (rows=40, cols=120, dirtyLines=40) | Rezi (native) | 85.6MB | 26.0MB | +16.7MB | +5.9MB | N/A | N/A | -| terminal-virtual-list (items=100000, viewport=40) | Rezi (native) | 133.1MB | 67.4MB | +43.6MB | +19.4MB | N/A | N/A | -| terminal-table (rows=40, cols=8) | Rezi (native) | 76.1MB | 23.0MB | +13.8MB | +6.3MB | N/A | N/A | +| terminal-rerender | Rezi (native) | 76.1MB | 16.2MB | +8.7MB | +6.1MB | N/A | N/A | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=1) | Rezi (native) | 90.1MB | 24.3MB | +22.3MB | +12.3MB | N/A | N/A | +| terminal-frame-fill (rows=40, cols=120, dirtyLines=40) | Rezi (native) | 97.3MB | 30.7MB | +27.4MB | +21.2MB | N/A | N/A | +| terminal-virtual-list (items=100000, viewport=40) | Rezi (native) | 150.8MB | 70.4MB | +71.2MB | +28.5MB | N/A | N/A | +| terminal-table (rows=40, cols=8) | Rezi (native) | 90.9MB | 27.9MB | +20.6MB | +14.8MB | N/A | N/A | diff --git a/docs/dev/perf-regressions.md b/docs/dev/perf-regressions.md index 9869d1f8..02057d51 100644 --- a/docs/dev/perf-regressions.md +++ b/docs/dev/perf-regressions.md @@ -79,7 +79,7 @@ Baseline path: Update process: -1. Run `npm run bench:ci -- --output-dir benchmarks/ci-baseline`. +1. Prefer CI-host refresh: download `perf-regression-artifacts` from the PR/job run and copy `results.json`/`results.md` into `benchmarks/ci-baseline/`. 2. Re-run compare against the updated baseline to verify zero hard regressions. 3. Keep threshold changes (`benchmarks/ci-baseline/config.json`) explicit in the same PR only when justified. 4. In PR description, include why baseline movement is intentional (feature/perf tradeoff, infra change, or expected engine shift).