Skip to content

Commit 736c050

Browse files
chirag-brunochirag-bruno
authored
feat: add benchmark framework for collection mount performance (usebruno#7915)
1 parent b79349b commit 736c050

16 files changed

Lines changed: 928 additions & 1 deletion

‎.github/actions/tests/run-benchmark-tests/action.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: 'Run Benchmark Tests'
2+
description: 'Run Playwright benchmark tests and compare against baseline'
3+
inputs:
4+
os:
5+
description: 'Operating system (ubuntu, macos, windows)'
6+
default: 'ubuntu'
7+
update-baseline:
8+
description: 'Update baseline instead of comparing'
9+
default: 'false'
10+
runs:
11+
using: 'composite'
12+
steps:
13+
- name: Run Benchmark Tests (Ubuntu)
14+
if: inputs.os == 'ubuntu'
15+
shell: bash
16+
run: xvfb-run npm run test:benchmark
17+
18+
- name: Run Benchmark Tests
19+
if: inputs.os != 'ubuntu'
20+
shell: bash
21+
run: npm run test:benchmark
22+
23+
- name: Update Baseline
24+
if: inputs.update-baseline == 'true'
25+
shell: bash
26+
run: >-
27+
node tests/benchmarks/utils/compare.js
28+
--results tests/benchmarks/results/mounting.json
29+
--baseline tests/benchmarks/mounting/baseline.${{ inputs.os }}.json
30+
--update-baseline
31+
32+
- name: Compare Against Baseline
33+
if: inputs.update-baseline != 'true'
34+
shell: bash
35+
run: >-
36+
node tests/benchmarks/utils/compare.js
37+
--results tests/benchmarks/results/mounting.json
38+
--baseline tests/benchmarks/mounting/baseline.${{ inputs.os }}.json

‎.github/workflows/benchmarks.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: Benchmarks
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
update-baseline:
6+
description: 'Update baseline with current results instead of comparing'
7+
type: boolean
8+
default: false
9+
pull_request:
10+
branches: [main, 'release/v*']
11+
12+
jobs:
13+
benchmark:
14+
name: Performance Benchmarks (${{ matrix.os }})
15+
timeout-minutes: 60
16+
runs-on: ${{ matrix.os }}
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
os: [ubuntu-24.04, macos-latest, windows-latest]
21+
include:
22+
- os: ubuntu-24.04
23+
os-name: ubuntu
24+
- os: macos-latest
25+
os-name: macos
26+
- os: windows-latest
27+
os-name: windows
28+
permissions:
29+
contents: write
30+
pull-requests: write
31+
steps:
32+
- uses: actions/checkout@v6
33+
34+
- name: Install System Dependencies (Ubuntu)
35+
if: matrix.os-name == 'ubuntu'
36+
run: |
37+
sudo apt-get update
38+
sudo apt-get --no-install-recommends install -y \
39+
libglib2.0-0 libnss3 libdbus-1-3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libgtk-3-0 libasound2t64 \
40+
xvfb
41+
42+
- name: Setup Node Dependencies
43+
uses: ./.github/actions/common/setup-node-deps
44+
45+
- name: Configure Chrome Sandbox
46+
if: matrix.os-name == 'ubuntu'
47+
run: |
48+
sudo chown root node_modules/electron/dist/chrome-sandbox
49+
sudo chmod 4755 node_modules/electron/dist/chrome-sandbox
50+
51+
- name: Run Benchmark Tests
52+
uses: ./.github/actions/tests/run-benchmark-tests
53+
with:
54+
os: ${{ matrix.os-name }}
55+
update-baseline: ${{ github.event.inputs.update-baseline || 'false' }}
56+
57+
- name: Upload Benchmark Results
58+
uses: actions/upload-artifact@v6
59+
if: ${{ !cancelled() }}
60+
with:
61+
name: benchmark-results-${{ matrix.os-name }}
62+
path: |
63+
tests/benchmarks/results/
64+
benchmark-report/
65+
retention-days: 30
66+
67+
- name: Commit Updated Baseline
68+
if: github.event.inputs.update-baseline == 'true'
69+
run: |
70+
git config user.name "github-actions[bot]"
71+
git config user.email "github-actions[bot]@users.noreply.github.com"
72+
git add tests/benchmarks/mounting/baseline.${{ matrix.os-name }}.json
73+
git diff --staged --quiet || git commit -m "chore: update ${{ matrix.os-name }} benchmark baseline" && git push
74+
75+
- name: Comment Benchmark Results on PR
76+
if: github.event_name == 'pull_request' && !cancelled()
77+
continue-on-error: true
78+
uses: actions/github-script@v7
79+
with:
80+
script: |
81+
const run = require('./tests/benchmarks/utils/pr-comment.js');
82+
await run({
83+
github,
84+
context,
85+
resultsPath: 'tests/benchmarks/results/mounting.json',
86+
baselinePath: 'tests/benchmarks/mounting/baseline.${{ matrix.os-name }}.json',
87+
title: 'Benchmark Results — Collection Mount (${{ matrix.os-name }})'
88+
});

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ skills-lock.json
5858
# Playwright
5959
/blob-report/
6060

61+
# Benchmark results (generated at runtime)
62+
tests/benchmarks/results/
63+
/benchmark-report/
64+
6165
# Development plan files
6266
CLAUDE.md
6367
AGENTS.md

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
"test:e2e": "playwright test --project=default",
8484
"test:e2e:ssl": "playwright test --project=ssl",
8585
"test:e2e:auth": "playwright test --project=auth",
86+
"test:benchmark": "playwright test --config=playwright.benchmark.config.ts",
8687
"lint": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" npx eslint",
8788
"lint:fix": "cross-env NODE_OPTIONS=\"--max_old_space_size=4096\" npx eslint --fix",
8889
"prepare": "husky"

‎playwright.benchmark.config.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import { defineConfig } from '@playwright/test';
2+
3+
export default defineConfig({
4+
fullyParallel: false,
5+
forbidOnly: !!process.env.CI,
6+
retries: 0,
7+
workers: 1,
8+
reporter: [
9+
['list'],
10+
['json', { outputFile: 'benchmark-report/results.json' }]
11+
],
12+
13+
use: {
14+
trace: 'off'
15+
},
16+
17+
projects: [
18+
{
19+
name: 'benchmarks',
20+
testDir: './tests/benchmarks',
21+
testMatch: '**/*.bench.ts'
22+
}
23+
],
24+
25+
webServer: [
26+
{
27+
command: 'npm run dev:web',
28+
url: 'http://localhost:3000',
29+
reuseExistingServer: !process.env.CI,
30+
timeout: 10 * 60 * 1000
31+
}
32+
],
33+
34+
timeout: 10 * 60 * 1000,
35+
expect: {
36+
timeout: 120_000
37+
}
38+
});

‎playwright.config.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ export default defineConfig({
2323
testDir: './tests',
2424
testIgnore: [
2525
'ssl/**', // custom CA certificate tests require separate server setup and certificate generation
26-
'auth/**' // auth tests have their own project
26+
'auth/**', // auth tests have their own project
27+
'benchmarks/**'
2728
]
2829
},
2930
{

‎tests/benchmarks/mounting/baseline.macos.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"thresholdPercent": 20,
3+
"entries": {
4+
"bru-50": {
5+
"mean": 2200,
6+
"p50": 1000
7+
},
8+
"bru-200": {
9+
"mean": 1300,
10+
"p50": 1100
11+
},
12+
"bru-500": {
13+
"mean": 3600,
14+
"p50": 3500
15+
},
16+
"bru-1000": {
17+
"mean": 9100,
18+
"p50": 9000
19+
},
20+
"bru-3000": {
21+
"mean": 185000,
22+
"p50": 183000
23+
},
24+
"yml-50": {
25+
"mean": 700,
26+
"p50": 650
27+
},
28+
"yml-200": {
29+
"mean": 1400,
30+
"p50": 1250
31+
},
32+
"yml-500": {
33+
"mean": 3900,
34+
"p50": 3700
35+
},
36+
"yml-1000": {
37+
"mean": 11700,
38+
"p50": 11900
39+
},
40+
"yml-3000": {
41+
"mean": 85000,
42+
"p50": 80000
43+
}
44+
}
45+
}

‎tests/benchmarks/mounting/baseline.ubuntu.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"thresholdPercent": 20,
3+
"entries": {
4+
"bru-50": {
5+
"mean": 1500,
6+
"p50": 700
7+
},
8+
"bru-200": {
9+
"mean": 1200,
10+
"p50": 1150
11+
},
12+
"bru-500": {
13+
"mean": 2900,
14+
"p50": 2900
15+
},
16+
"bru-1000": {
17+
"mean": 8000,
18+
"p50": 8000
19+
},
20+
"bru-3000": {
21+
"mean": 175000,
22+
"p50": 170000
23+
},
24+
"yml-50": {
25+
"mean": 600,
26+
"p50": 560
27+
},
28+
"yml-200": {
29+
"mean": 1200,
30+
"p50": 1200
31+
},
32+
"yml-500": {
33+
"mean": 3500,
34+
"p50": 3400
35+
},
36+
"yml-1000": {
37+
"mean": 10700,
38+
"p50": 10650
39+
},
40+
"yml-3000": {
41+
"mean": 85000,
42+
"p50": 80000
43+
}
44+
}
45+
}

‎tests/benchmarks/mounting/baseline.windows.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"thresholdPercent": 20,
3+
"entries": {
4+
"bru-50": {
5+
"mean": 2700,
6+
"p50": 800
7+
},
8+
"bru-200": {
9+
"mean": 1500,
10+
"p50": 1400
11+
},
12+
"bru-500": {
13+
"mean": 3500,
14+
"p50": 3500
15+
},
16+
"bru-1000": {
17+
"mean": 9500,
18+
"p50": 9400
19+
},
20+
"bru-3000": {
21+
"mean": 195000,
22+
"p50": 190000
23+
},
24+
"yml-50": {
25+
"mean": 600,
26+
"p50": 570
27+
},
28+
"yml-200": {
29+
"mean": 1350,
30+
"p50": 1300
31+
},
32+
"yml-500": {
33+
"mean": 3800,
34+
"p50": 3700
35+
},
36+
"yml-1000": {
37+
"mean": 11000,
38+
"p50": 11000
39+
},
40+
"yml-3000": {
41+
"mean": 90000,
42+
"p50": 88000
43+
}
44+
}
45+
}

0 commit comments

Comments
 (0)