Skip to content

Commit ac385d5

Browse files
davesnxdavesnx
committed
Simplfy all benchmarks
1 parent c89b675 commit ac385d5

9 files changed

Lines changed: 109 additions & 464 deletions

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 60 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,23 @@
1-
name: Benchmark
1+
name: Framework Comparison
22

33
on:
4-
# Run on PRs to compare performance
5-
pull_request:
6-
paths:
7-
- 'packages/**'
8-
- 'benchmark/**'
9-
# Manual trigger
4+
push:
5+
tags:
6+
- 'v*'
107
workflow_dispatch:
118
inputs:
12-
suite:
13-
description: 'Benchmark suite to run'
9+
frameworks:
10+
description: 'Frameworks to test (comma-separated, or "all")'
1411
required: false
1512
default: 'all'
16-
type: choice
17-
options:
18-
- all
19-
- micro
20-
- memory
21-
- streaming
22-
# Weekly scheduled run for tracking
23-
schedule:
24-
- cron: '0 0 * * 0' # Every Sunday at midnight
13+
scenarios:
14+
description: 'Scenarios to run (comma-separated, or "all")'
15+
required: false
16+
default: 'trivial,table100,table500'
2517

2618
jobs:
27-
benchmark-native:
28-
name: Native Benchmarks
19+
benchmark-frameworks:
20+
name: Compare Frameworks
2921
runs-on: ubuntu-latest
3022

3123
steps:
@@ -37,106 +29,69 @@ jobs:
3729
ocaml-compiler: 5.2.x
3830
dune-cache: true
3931

40-
- name: Install dependencies
41-
run: opam install . --deps-only --with-test -y
32+
- name: Setup Node.js
33+
uses: actions/setup-node@v4
34+
with:
35+
node-version: '20'
4236

43-
- name: Build benchmark scenarios
44-
run: opam exec -- dune build benchmark/scenarios/benchmark_scenarios.a
37+
- name: Setup Bun
38+
uses: oven-sh/setup-bun@v1
4539

46-
- name: Build benchmark executables
40+
- name: Install wrk
4741
run: |
48-
opam exec -- dune build benchmark/micro/micro_bench.exe
49-
opam exec -- dune build benchmark/memory/memory_bench.exe
50-
opam exec -- dune build benchmark/streaming/streaming_bench.exe
42+
sudo apt-get update
43+
sudo apt-get install -y wrk
5144
52-
- name: Run Memory Benchmark
53-
id: memory
54-
run: |
55-
echo "## Memory Benchmark Results" >> $GITHUB_STEP_SUMMARY
56-
echo '```' >> $GITHUB_STEP_SUMMARY
57-
opam exec -- _build/default/benchmark/memory/memory_bench.exe 2>&1 | tee memory-results.txt
58-
cat memory-results.txt >> $GITHUB_STEP_SUMMARY
59-
echo '```' >> $GITHUB_STEP_SUMMARY
60-
61-
- name: Run Streaming Benchmark
62-
id: streaming
63-
run: |
64-
echo "## Streaming Benchmark Results" >> $GITHUB_STEP_SUMMARY
65-
echo '```' >> $GITHUB_STEP_SUMMARY
66-
opam exec -- _build/default/benchmark/streaming/streaming_bench.exe 2>&1 | tee streaming-results.txt
67-
cat streaming-results.txt >> $GITHUB_STEP_SUMMARY
68-
echo '```' >> $GITHUB_STEP_SUMMARY
69-
70-
- name: Run Microbenchmark (Quick)
71-
id: micro
72-
run: |
73-
echo "## Microbenchmark Results (Table Suite)" >> $GITHUB_STEP_SUMMARY
74-
echo '```' >> $GITHUB_STEP_SUMMARY
75-
opam exec -- _build/default/benchmark/micro/micro_bench.exe table 2>&1 | tee micro-results.txt
76-
cat micro-results.txt >> $GITHUB_STEP_SUMMARY
77-
echo '```' >> $GITHUB_STEP_SUMMARY
45+
- name: Install OCaml dependencies
46+
run: opam install . --deps-only -y
7847

79-
- name: Upload Results
80-
uses: actions/upload-artifact@v4
81-
with:
82-
name: benchmark-results-${{ github.sha }}
83-
path: |
84-
memory-results.txt
85-
streaming-results.txt
86-
micro-results.txt
87-
retention-days: 30
88-
89-
benchmark-compare:
90-
name: Compare with Base Branch
91-
runs-on: ubuntu-latest
92-
if: github.event_name == 'pull_request'
48+
- name: Build native server
49+
run: opam exec -- dune build benchmark/native/server.exe --profile=release
9350

94-
steps:
95-
- uses: actions/checkout@v4
96-
with:
97-
fetch-depth: 0
51+
- name: Install JS framework dependencies
52+
working-directory: benchmark/frameworks
53+
run: npm install
9854

99-
- name: Setup OCaml
100-
uses: ocaml/setup-ocaml@v3
101-
with:
102-
ocaml-compiler: 5.2.x
103-
dune-cache: true
55+
- name: Install runner dependencies
56+
working-directory: benchmark/runner
57+
run: npm install
10458

105-
- name: Install dependencies
106-
run: opam install . --deps-only --with-test -y
59+
- name: Start native server
60+
run: |
61+
opam exec -- _build/default/benchmark/native/server.exe &
62+
sleep 2
10763
108-
- name: Benchmark PR branch
64+
- name: Run framework comparison
65+
working-directory: benchmark/runner
10966
run: |
110-
opam exec -- dune build benchmark/memory/memory_bench.exe
111-
opam exec -- _build/default/benchmark/memory/memory_bench.exe --json > pr-results.json
67+
FRAMEWORKS="${{ github.event.inputs.frameworks || 'all' }}"
68+
SCENARIOS="${{ github.event.inputs.scenarios || 'trivial,table100,table500' }}"
11269
113-
- name: Checkout base branch
114-
run: git checkout ${{ github.base_ref }}
70+
ARGS=""
71+
if [ "$FRAMEWORKS" != "all" ]; then
72+
ARGS="$ARGS --frameworks $FRAMEWORKS"
73+
fi
74+
if [ "$SCENARIOS" != "all" ]; then
75+
ARGS="$ARGS --scenarios $SCENARIOS"
76+
fi
11577
116-
- name: Benchmark base branch
117-
run: |
118-
opam exec -- dune clean
119-
opam exec -- dune build benchmark/memory/memory_bench.exe
120-
opam exec -- _build/default/benchmark/memory/memory_bench.exe --json > base-results.json
121-
continue-on-error: true
78+
node runner.mjs $ARGS
79+
80+
- name: Upload results
81+
uses: actions/upload-artifact@v4
82+
with:
83+
name: framework-comparison-${{ github.sha }}
84+
path: benchmark/runner/results/
85+
retention-days: 90
12286

123-
- name: Compare Results
87+
- name: Add results to summary
12488
run: |
125-
echo "## Performance Comparison" >> $GITHUB_STEP_SUMMARY
89+
echo "## Framework Comparison Results" >> $GITHUB_STEP_SUMMARY
12690
echo "" >> $GITHUB_STEP_SUMMARY
127-
if [ -f base-results.json ] && [ -f pr-results.json ]; then
128-
echo "Comparing memory allocations between base and PR..." >> $GITHUB_STEP_SUMMARY
129-
# Simple comparison - in production you'd use a proper tool
130-
echo "Base branch results:" >> $GITHUB_STEP_SUMMARY
131-
echo '```json' >> $GITHUB_STEP_SUMMARY
132-
cat base-results.json >> $GITHUB_STEP_SUMMARY
133-
echo '```' >> $GITHUB_STEP_SUMMARY
134-
echo "" >> $GITHUB_STEP_SUMMARY
135-
echo "PR branch results:" >> $GITHUB_STEP_SUMMARY
136-
echo '```json' >> $GITHUB_STEP_SUMMARY
137-
cat pr-results.json >> $GITHUB_STEP_SUMMARY
138-
echo '```' >> $GITHUB_STEP_SUMMARY
91+
LATEST=$(ls -t benchmark/runner/results/*.md 2>/dev/null | head -1)
92+
if [ -n "$LATEST" ]; then
93+
cat "$LATEST" >> $GITHUB_STEP_SUMMARY
13994
else
140-
echo "⚠️ Could not compare - base branch benchmark may not exist" >> $GITHUB_STEP_SUMMARY
95+
echo "No results generated" >> $GITHUB_STEP_SUMMARY
14196
fi
14297

‎.github/workflows/ci.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,26 @@ jobs:
146146
- name: Run benchmarks
147147
run: make bench
148148

149+
- name: Run benchmarks as JSON
150+
if: matrix.os == 'ubuntu-latest' && matrix.ocaml-compiler == '5.2.0'
151+
run: make bench-json
152+
153+
- name: Store benchmark result
154+
if: matrix.os == 'ubuntu-latest' && matrix.ocaml-compiler == '5.2.0'
155+
uses: benchmark-action/github-action-benchmark@v1
156+
with:
157+
name: server-reason-react Benchmarks
158+
tool: 'customBiggerIsBetter'
159+
output-file-path: bench_results.json
160+
github-token: ${{ secrets.GITHUB_TOKEN }}
161+
auto-push: ${{ github.ref == 'refs/heads/main' }}
162+
gh-pages-branch: gh-pages
163+
benchmark-data-dir-path: dev/bench
164+
comment-always: true
165+
fail-on-alert: true
166+
comment-on-alert: true
167+
alert-comment-cc-users: '@davesnx'
168+
149169
- name: Install dune-release
150170
if: startsWith(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-latest' && matrix.ocaml-compiler == '5.2.0'
151171
run: opam install dune-release -y

‎Makefile‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -143,18 +143,9 @@ bench-json: build-bench ## Run benchmarks with JSON output for CI
143143
bench-watch: build-bench ## Run benchmark in watch mode
144144
@$(DUNE) exec benchmark/bench.exe --profile=release --display-separate-messages --no-print-directory --watch
145145

146-
.PHONY: bench-core
147-
bench-core: ## Run Core_bench benchmarks (more detailed)
148-
$(DUNE) build --profile=release benchmark/main.exe
149-
@$(DUNE) exec benchmark/main.exe --profile=release --display-separate-messages --no-print-directory
150-
151-
.PHONY: bench-once
152-
bench-once: ## Run benchmark once (allocation analysis)
153-
@$(DUNE) exec _build/default/benchmark/once.exe
154-
155-
.PHONY: bench-once-watch
156-
bench-once-watch: ## Run benchmark once in watch mode
157-
@$(DUNE) exec _build/default/benchmark/once.exe --watch
146+
.PHONY: bench-allocation
147+
bench-allocation: ## Run allocation analysis
148+
@$(DUNE) exec benchmark/allocation.exe --profile=release
158149

159150
container_name = server-reason-react-demo
160151
current_hash = $(shell git rev-parse HEAD | cut -c1-7)

benchmark/once.ml renamed to benchmark/allocation.ml

File renamed without changes.

‎benchmark/bench.ml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,17 +58,41 @@ let () =
5858
measure_benchmark ~name:"width/10" (fun () -> ReactDOM.renderToStaticMarkup (WideTree.Wide10.make ()));
5959
measure_benchmark ~name:"width/100" (fun () -> ReactDOM.renderToStaticMarkup (WideTree.Wide100.make ()));
6060
measure_benchmark ~name:"width/500" (fun () -> ReactDOM.renderToStaticMarkup (WideTree.Wide500.make ()));
61+
measure_benchmark ~name:"width/1000" (fun () -> ReactDOM.renderToStaticMarkup (WideTree.Wide1000.make ()));
6162
measure_benchmark ~name:"table/10" (fun () -> ReactDOM.renderToStaticMarkup (Table.Table10.make ()));
63+
measure_benchmark ~name:"table/50" (fun () -> ReactDOM.renderToStaticMarkup (Table.Table50.make ()));
6264
measure_benchmark ~name:"table/100" (fun () -> ReactDOM.renderToStaticMarkup (Table.Table100.make ()));
6365
measure_benchmark ~name:"table/500" (fun () -> ReactDOM.renderToStaticMarkup (Table.Table500.make ()));
6466
measure_benchmark ~name:"props/small" (fun () -> ReactDOM.renderToStaticMarkup (PropsHeavy.Small.make ()));
6567
measure_benchmark ~name:"props/medium" (fun () -> ReactDOM.renderToStaticMarkup (PropsHeavy.Medium.make ()));
6668
measure_benchmark ~name:"props/large" (fun () -> ReactDOM.renderToStaticMarkup (PropsHeavy.Large.make ()));
6769
measure_benchmark ~name:"realworld/ecommerce24" (fun () ->
6870
ReactDOM.renderToStaticMarkup (Ecommerce.Products24.make ()));
71+
measure_benchmark ~name:"realworld/ecommerce48" (fun () ->
72+
ReactDOM.renderToStaticMarkup (Ecommerce.Products48.make ()));
6973
measure_benchmark ~name:"realworld/dashboard" (fun () -> ReactDOM.renderToStaticMarkup (Dashboard.make ()));
7074
measure_benchmark ~name:"realworld/blog50" (fun () -> ReactDOM.renderToStaticMarkup (Blog.Blog50.make ()));
7175
measure_benchmark ~name:"realworld/form" (fun () -> ReactDOM.renderToStaticMarkup (Form.make ()));
76+
measure_benchmark ~name:"primitive/React.string" (fun () -> ReactDOM.renderToStaticMarkup (React.string "Hello"));
77+
measure_benchmark ~name:"primitive/React.int" (fun () -> ReactDOM.renderToStaticMarkup (React.int 42));
78+
measure_benchmark ~name:"primitive/React.null" (fun () -> ReactDOM.renderToStaticMarkup React.null);
79+
measure_benchmark ~name:"primitive/createElement_empty" (fun () ->
80+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] []));
81+
measure_benchmark ~name:"primitive/createElement_children" (fun () ->
82+
let children = List.init 10 (fun i -> React.string (string_of_int i)) in
83+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] children));
84+
measure_benchmark ~name:"primitive/React.array_10" (fun () ->
85+
let arr = Array.init 10 (fun i -> React.string (string_of_int i)) in
86+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] [ React.array arr ]));
87+
measure_benchmark ~name:"primitive/React.array_100" (fun () ->
88+
let arr = Array.init 100 (fun i -> React.string (string_of_int i)) in
89+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] [ React.array arr ]));
90+
measure_benchmark ~name:"primitive/React.list_10" (fun () ->
91+
let lst = List.init 10 (fun i -> React.string (string_of_int i)) in
92+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] [ React.list lst ]));
93+
measure_benchmark ~name:"primitive/React.list_100" (fun () ->
94+
let lst = List.init 100 (fun i -> React.string (string_of_int i)) in
95+
ReactDOM.renderToStaticMarkup (React.createElement "div" [] [ React.list lst ]));
7296
]
7397
in
7498

‎benchmark/dune‎

Lines changed: 2 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
1-
; Main benchmark dune file
2-
3-
; CI benchmark runner - simple, JSON-compatible output for github-action-benchmark
4-
51
(executable
62
(name bench)
73
(modules bench)
@@ -13,27 +9,9 @@
139
(preprocess
1410
(pps server-reason-react.ppx)))
1511

16-
; Legacy executables (keeping for compatibility)
17-
18-
(executable
19-
(name main)
20-
(modules main)
21-
(libraries
22-
base
23-
core
24-
unix
25-
core_unix
26-
core_unix.command_unix
27-
core_bench
28-
demo_shared_native
29-
server-reason-react.react
30-
server-reason-react.reactDom)
31-
(preprocess
32-
(pps server-reason-react.ppx)))
33-
3412
(executable
35-
(name once)
36-
(modules once)
13+
(name allocation)
14+
(modules allocation)
3715
(libraries
3816
unix
3917
lwt
@@ -55,8 +33,3 @@
5533
(with-stdout-to
5634
bench_results.json
5735
(run ./bench.exe --json))))
58-
59-
(rule
60-
(alias bench-once)
61-
(action
62-
(run ./once.exe)))

‎benchmark/main.re‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎benchmark/micro/dune‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)