Skip to content

Commit 70eb8b4

Browse files
aduh95aduh95
committed
tools: add GHA benchmark runner
Signed-off-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent a0263fe commit 70eb8b4

2 files changed

Lines changed: 209 additions & 1 deletion

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
name: Benchmark
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
repo:
7+
type: string
8+
description: GitHub repository to fetch from (default to the current repo)
9+
pr_id:
10+
type: number
11+
required: true
12+
description: The PR to test
13+
commit:
14+
required: true
15+
type: string
16+
description: The expect HEAD of the PR
17+
category:
18+
required: true
19+
type: string
20+
description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
21+
filter:
22+
type: string
23+
description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
24+
runs:
25+
type: number
26+
default: 30
27+
description: How many times to repeat each benchmark
28+
29+
permissions:
30+
contents: read
31+
32+
jobs:
33+
build:
34+
strategy:
35+
fail-fast: true
36+
matrix:
37+
include:
38+
- runner: ubuntu-24.04
39+
system: x86_64-linux
40+
- runner: ubuntu-24.04-arm
41+
system: aarch64-linux
42+
- runner: macos-15-intel
43+
system: x86_64-darwin
44+
- runner: macos-latest
45+
system: aarch64-darwin
46+
name: '${{ matrix.system }}: with shared libraries'
47+
runs-on: ${{ matrix.runner }}
48+
steps:
49+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
50+
with:
51+
repository: ${{ inputs.repo || github.repository }}
52+
ref: refs/pull/${{ inputs.pr_id }}/merge
53+
persist-credentials: false
54+
fetch-depth: 2
55+
56+
- name: Validate PR head and roll back to base commit
57+
run: |
58+
[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
59+
git reset HEAD^ --hard
60+
env:
61+
EXPECTED_SHA: ${{ inputs.commit }}
62+
63+
- uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3
64+
with:
65+
extra_nix_config: sandbox = true
66+
67+
- uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17
68+
with:
69+
# We do not pass any `authToken` to avoid polluting the cache with potentially untrusted code.
70+
name: nodejs
71+
72+
- name: Configure sccache
73+
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
74+
with:
75+
script: |
76+
core.exportVariable('SCCACHE_GHA_VERSION', 'on');
77+
core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
78+
core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
79+
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
80+
81+
- name: Build Node.js on the base commit
82+
run: |
83+
nix-shell \
84+
-I nixpkgs=./tools/nix/pkgs.nix \
85+
--pure --keep TAR_DIR --keep FLAKY_TESTS \
86+
--keep SCCACHE_GHA_ENABLED --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
87+
--arg useSeparateDerivationForV8 true \
88+
--arg loadJSBuiltinsDynamically false \
89+
--arg ccache "${NIX_SCCACHE:-null}" \
90+
--arg devTools '[]' \
91+
--arg benchmarkTools '[]' \
92+
--run '
93+
make build-ci -j4 V=1
94+
'
95+
mv out/Release/node base_node
96+
97+
- name: Checkout the merge commit
98+
run: git reset FETCH_HEAD --hard
99+
100+
- name: Re-build Node.js on the merge commit
101+
# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
102+
if: ${{ github.event_name == 'workflow_dispatch' }}
103+
run: |
104+
nix-shell \
105+
-I nixpkgs=./tools/nix/pkgs.nix \
106+
--pure \
107+
--arg useSeparateDerivationForV8 true \
108+
--arg loadJSBuiltinsDynamically false \
109+
--arg ccache 'null' \
110+
--arg devTools '[]' \
111+
--arg benchmarkTools '[]' \
112+
--run '
113+
make -j4 V=1
114+
'
115+
116+
- name: Run benchmark
117+
if: ${{ github.event_name == 'workflow_dispatch' }}
118+
run: |
119+
nix-shell \
120+
-I nixpkgs=./tools/nix/pkgs.nix \
121+
--pure --keep FILTER --keep LC_ALL --keep LANG \
122+
--arg loadJSBuiltinsDynamically false \
123+
--arg ccache 'null' \
124+
--arg icu 'null' \
125+
--arg sharedLibDeps '{}' \
126+
--arg devTools '[]' \
127+
--run '
128+
set -o pipefail
129+
./base_node benchmark/compare.js \
130+
--filter "$FILTER" \
131+
--runs ${{ inputs.runs }} \
132+
--old ./base_node --new ./node \
133+
-- ${{ inputs.category }} \
134+
| tee /dev/stderr \
135+
> ${{ matrix.system }}.csv
136+
echo "> [!WARNING] "
137+
echo "> Do not take GHA benchmark results as face value, always confirm them"
138+
echo "> using a dedicated machine, e.g. Jenkins CI."
139+
echo
140+
echo "Benchmark results:"
141+
echo
142+
echo '"'"'```'"'"'
143+
Rscript benchmark/compare.R < ${{ matrix.system }}.csv
144+
echo '"'"'```'"'"'
145+
echo
146+
echo "> [!WARNING] "
147+
echo "> Do not take GHA benchmark results as face value, always confirm them"
148+
echo "> using a dedicated machine, e.g. Jenkins CI."
149+
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
150+
env:
151+
FILTER: ${{ inputs.filter }}
152+
153+
- name: Upload raw benchmark results
154+
if: ${{ github.event_name == 'workflow_dispatch' }}
155+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
156+
with:
157+
name: csv-${{ matrix.system }}
158+
path: ${{ matrix.system }}.csv
159+
160+
aggregate-benchmark-results:
161+
needs: build
162+
name: Aggregate benchmark results
163+
if: ${{ github.event_name == 'workflow_dispatch' }}
164+
runs-on: ubuntu-latest
165+
steps:
166+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
167+
with:
168+
persist-credentials: false
169+
sparse-checkout: |
170+
benchmark/*.R
171+
tools/nix/*.nix
172+
*.nix
173+
sparse-checkout-cone-mode: false
174+
175+
- name: Download benchmark raw results
176+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
177+
with:
178+
pattern: csv-*
179+
merge-multiple: true
180+
path: raw-results
181+
182+
- uses: cachix/install-nix-action@96951a368ba55167b55f1c916f7d416bac6505fe # v31.10.3
183+
with:
184+
extra_nix_config: sandbox = true
185+
186+
- name: Benchmark results
187+
run: |
188+
nix-shell \
189+
-I nixpkgs=./tools/nix/pkgs.nix \
190+
--pure --keep LC_ALL --keep LANG \
191+
-E '(import <nixpkgs> {}).mkShell { buildInputs = import ./tools/nix/benchmarkTools.nix { withHttpBenchmarkDeps = false; }; }' \
192+
--run '
193+
echo "> [!WARNING] "
194+
echo "> Do not take GHA benchmark results as face value, always confirm them"
195+
echo "> using a dedicated machine, e.g. Jenkins CI."
196+
echo
197+
echo "Benchmark results:"
198+
echo
199+
echo '"'"'```'"'"'
200+
awk "FNR==1 && NR!=1{next;}{print}" raw-results/*.csv | Rscript benchmark/compare.R
201+
echo '"'"'```'"'"'
202+
echo
203+
echo "> [!WARNING] "
204+
echo "> Do not take GHA benchmark results as face value, always confirm them"
205+
echo "> using a dedicated machine, e.g. Jenkins CI."
206+
' | tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"

‎tools/nix/benchmarkTools.nix‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
pkgs ? import ./pkgs.nix { },
3+
withHttpBenchmarkDeps ? true,
34
}:
45
[
56
pkgs.R
67
pkgs.rPackages.ggplot2
78
pkgs.rPackages.plyr
8-
pkgs.wrk
99
]
10+
++ pkgs.lib.optional withHttpBenchmarkDeps pkgs.wrk
11+
++ pkgs.lib.optional pkgs.stdenv.buildPlatform.isLinux pkgs.glibcLocales

0 commit comments

Comments
 (0)