Skip to content

Commit e6218f0

Browse files
skylenetskylenet
authored
feat: add reusable GitHub Actions composite action for running benchmarks (#71)
1 parent bea269a commit e6218f0

13 files changed

Lines changed: 506 additions & 119 deletions

File tree

‎.github/workflows/ci.action.yaml‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: Check - CI (Github Action)
2+
3+
on:
4+
pull_request:
5+
branches: [master]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
pr-benchmark:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout repository
16+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
17+
18+
- name: Run Benchmarkoor Test
19+
uses: ./
20+
with:
21+
github-token: ${{ secrets.GITHUB_TOKEN }}
22+
git-ref: ${{ github.event.pull_request.head.sha }}
23+
#run-args: '--log-level=debug'
24+
run-config: |
25+
global:
26+
log_level: debug
27+
client_logs_to_stdout: true
28+
cleanup_on_start: true
29+
30+
benchmark:
31+
system_resource_collection_enabled: true
32+
tests:
33+
filter: "bn128"
34+
source:
35+
eest_fixtures:
36+
github_repo: ethereum/execution-spec-tests
37+
github_release: benchmark@v0.0.5
38+
39+
client:
40+
config:
41+
drop_memory_caches: "steps"
42+
resource_limits:
43+
# The CPU freq limit might not work on public github runners.
44+
# cpu_freq: "2100MHz"
45+
# cpu_turboboost: false
46+
# cpu_freq_governor: performance
47+
cpuset_count: 2
48+
memory: "4g"
49+
swap_disabled: true
50+
51+
instances:
52+
- id: geth
53+
client: geth

‎action.yaml‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
name: 'Benchmarkoor'
2+
description: 'Run Ethereum execution client benchmarks using benchmarkoor'
3+
author: 'ethpandaops'
4+
branding:
5+
icon: 'activity'
6+
color: 'gray-dark'
7+
8+
inputs:
9+
github-token:
10+
description: 'GitHub token for API access'
11+
required: true
12+
13+
image:
14+
description: 'Docker image for benchmarkoor (e.g., ghcr.io/ethpandaops/benchmarkoor:master). If not provided, will build locally.'
15+
required: false
16+
default: ''
17+
18+
git-ref:
19+
description: 'Git branch or commit hash to build from. Only used when image is not provided. Defaults to master.'
20+
required: false
21+
default: ''
22+
23+
upload-artifacts:
24+
description: 'Whether to upload run results as GitHub artifacts'
25+
required: false
26+
default: 'false'
27+
28+
run-config-url:
29+
description: 'URL for config file to download and pass via --config'
30+
required: false
31+
default: ''
32+
33+
run-config:
34+
description: 'Raw YAML config content to pass via --config (appended after URL config)'
35+
required: false
36+
default: ''
37+
38+
run-args:
39+
description: 'Extra flags passed to benchmarkoor run command'
40+
required: false
41+
default: ''
42+
43+
runs:
44+
using: 'composite'
45+
steps:
46+
- name: Build Docker image locally
47+
if: inputs.image == ''
48+
shell: bash
49+
run: |
50+
GIT_REF="${{ inputs.git-ref }}"
51+
if [ -z "$GIT_REF" ]; then
52+
GIT_REF="master"
53+
fi
54+
55+
echo "Cloning https://github.com/ethpandaops/benchmarkoor.git at ref $GIT_REF"
56+
git clone https://github.com/ethpandaops/benchmarkoor.git benchmarkoor-build
57+
cd benchmarkoor-build
58+
git checkout "$GIT_REF"
59+
60+
echo "Building Docker image..."
61+
docker build -t benchmarkoor:local .
62+
cd ..
63+
64+
rm -rf benchmarkoor-build
65+
66+
- name: Determine which image to use
67+
id: determine-image
68+
shell: bash
69+
run: |
70+
if [ -n "${{ inputs.image }}" ]; then
71+
IMAGE="${{ inputs.image }}"
72+
echo "Using provided Docker image: ${IMAGE}"
73+
else
74+
IMAGE="benchmarkoor:local"
75+
echo "Using locally built image: ${IMAGE}"
76+
fi
77+
echo "image=${IMAGE}" >> $GITHUB_OUTPUT
78+
79+
if [ -z "$IMAGE" ]; then
80+
echo "ERROR: Docker image not specified"
81+
exit 1
82+
fi
83+
84+
if [ "$IMAGE" != "benchmarkoor:local" ]; then
85+
echo "Pulling Docker image: ${IMAGE}"
86+
docker pull "${IMAGE}" || {
87+
echo "Failed to pull Docker image: ${IMAGE}"
88+
exit 1
89+
}
90+
fi
91+
92+
- name: Get Job ID from GH API
93+
id: get-job-id
94+
shell: bash
95+
env:
96+
GH_TOKEN: ${{ inputs.github-token }}
97+
run: |
98+
jobs=$(gh api -F per_page=100 -X GET repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs)
99+
job_id=$(echo "$jobs" | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}" and .status=="in_progress") | .id')
100+
echo "job_id=${job_id}" >> "$GITHUB_OUTPUT"
101+
102+
- name: Run Benchmarkoor
103+
id: run-benchmarkoor
104+
shell: bash
105+
env:
106+
INPUT_RUN_CONFIG: ${{ inputs.run-config }}
107+
INPUT_RUN_CONFIG_URL: ${{ inputs.run-config-url }}
108+
run: |
109+
# Remove any leftover containers from a previous run
110+
docker rm -f benchmarkoor-run 2>/dev/null || true
111+
docker rm -f benchmarkoor-md 2>/dev/null || true
112+
113+
mkdir -p ${{ runner.temp }}/results
114+
115+
IMAGE="${{ steps.determine-image.outputs.image }}"
116+
echo "Running benchmarkoor with Docker image: ${IMAGE}"
117+
118+
# Download URL config if provided
119+
if [ -n "$INPUT_RUN_CONFIG_URL" ]; then
120+
echo "Downloading config from ${INPUT_RUN_CONFIG_URL}"
121+
curl -fsSL "${INPUT_RUN_CONFIG_URL}" -o "${{ runner.temp }}/benchmarkoor-config-url.yaml"
122+
fi
123+
124+
# Write inline config if provided
125+
if [ -n "$INPUT_RUN_CONFIG" ]; then
126+
cat <<'CONFIGEOF' > "${{ runner.temp }}/benchmarkoor-config-inline.yaml"
127+
${{ inputs.run-config }}
128+
CONFIGEOF
129+
fi
130+
131+
# Build docker run command
132+
DOCKER_CMD="docker run --network=host --rm --name=benchmarkoor-run"
133+
DOCKER_CMD="$DOCKER_CMD -v /var/run/docker.sock:/var/run/docker.sock"
134+
DOCKER_CMD="$DOCKER_CMD -v ${{ runner.temp }}/results:/app/results"
135+
DOCKER_CMD="$DOCKER_CMD -v ${{ runner.temp }}:/app/configs"
136+
DOCKER_CMD="$DOCKER_CMD -v /tmp:/tmp"
137+
DOCKER_CMD="$DOCKER_CMD --cap-add=SYS_ADMIN"
138+
DOCKER_CMD="$DOCKER_CMD -v /proc/sys/vm/drop_caches:/host_drop_caches"
139+
DOCKER_CMD="$DOCKER_CMD -v /sys/devices/system/cpu:/host_sys_cpu"
140+
DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_GLOBAL_DROP_CACHES_PATH=/host_drop_caches"
141+
DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_GLOBAL_CPU_SYSFS_PATH=/host_sys_cpu"
142+
DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_BENCHMARK_RESULTS_DIR=/app/results"
143+
DOCKER_CMD="$DOCKER_CMD -e BENCHMARKOOR_BENCHMARK_RESULTS_OWNER=$(id -u):$(id -g)"
144+
DOCKER_CMD="$DOCKER_CMD ${IMAGE}"
145+
DOCKER_CMD="$DOCKER_CMD run"
146+
147+
# Add config files in order: URL config, inline config, override config (last wins)
148+
if [ -f "${{ runner.temp }}/benchmarkoor-config-url.yaml" ]; then
149+
DOCKER_CMD="$DOCKER_CMD --config=/app/configs/benchmarkoor-config-url.yaml"
150+
fi
151+
152+
if [ -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" ]; then
153+
DOCKER_CMD="$DOCKER_CMD --config=/app/configs/benchmarkoor-config-inline.yaml"
154+
fi
155+
156+
# Add GitHub Actions context as metadata labels
157+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.run_id=${{ github.run_id }}"
158+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.run_number=${{ github.run_number }}"
159+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.job=${{ github.job }}"
160+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.job_id=${{ steps.get-job-id.outputs.job_id }}"
161+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.repository=${{ github.repository }}"
162+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.workflow=\"${{ github.workflow }}\""
163+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.sha=${{ github.sha }}"
164+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.actor=${{ github.actor }}"
165+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.event_name=${{ github.event_name }}"
166+
DOCKER_CMD="$DOCKER_CMD --metadata.label=github.ref=${{ github.ref }}"
167+
168+
# Append extra run args if provided
169+
if [ -n "${{ inputs.run-args }}" ]; then
170+
DOCKER_CMD="$DOCKER_CMD ${{ inputs.run-args }}"
171+
fi
172+
173+
# Execute the command
174+
echo "Running: $DOCKER_CMD"
175+
set +e
176+
eval $DOCKER_CMD &
177+
wait $!
178+
DOCKER_EXIT_CODE=$?
179+
set -e
180+
181+
# Find run directories
182+
RUN_DIRS=""
183+
if [ -d "${{ runner.temp }}/results/runs" ]; then
184+
RUN_DIRS=$(find "${{ runner.temp }}/results/runs" -mindepth 1 -maxdepth 1 -type d | sort | tr '\n' ' ')
185+
fi
186+
187+
echo "run-dirs=${RUN_DIRS}" >> $GITHUB_OUTPUT
188+
189+
if [ $DOCKER_EXIT_CODE -ne 0 ]; then
190+
echo "Benchmarkoor failed with exit code $DOCKER_EXIT_CODE"
191+
exit $DOCKER_EXIT_CODE
192+
fi
193+
194+
echo "Benchmarkoor completed successfully"
195+
196+
- name: Generate markdown summaries
197+
if: always() && steps.run-benchmarkoor.outputs.run-dirs != ''
198+
shell: bash
199+
run: |
200+
IMAGE="${{ steps.determine-image.outputs.image }}"
201+
202+
for RUN_DIR_HOST in ${{ steps.run-benchmarkoor.outputs.run-dirs }}; do
203+
RUN_DIR_NAME=$(basename "$RUN_DIR_HOST")
204+
CONTAINER_RUN_DIR="/app/results/runs/${RUN_DIR_NAME}"
205+
206+
echo "Generating summary for ${RUN_DIR_NAME}"
207+
208+
docker run --rm --name=benchmarkoor-md \
209+
-v "${{ runner.temp }}/results:/app/results" \
210+
"${IMAGE}" \
211+
generate-markdown-summary \
212+
--run-dir="${CONTAINER_RUN_DIR}" \
213+
--output="${CONTAINER_RUN_DIR}/summary.md"
214+
215+
if [ -f "${RUN_DIR_HOST}/summary.md" ]; then
216+
cat "${RUN_DIR_HOST}/summary.md" >> "$GITHUB_STEP_SUMMARY"
217+
echo "" >> "$GITHUB_STEP_SUMMARY"
218+
fi
219+
done
220+
221+
- name: Create results archive
222+
if: always() && inputs.upload-artifacts == 'true'
223+
shell: bash
224+
run: |
225+
if [ -d "${{ runner.temp }}/results" ]; then
226+
tar -czf "${{ runner.temp }}/benchmarkoor-results.tar.gz" -C "${{ runner.temp }}" results
227+
fi
228+
229+
- name: Upload artifacts
230+
uses: actions/upload-artifact@v4
231+
if: always() && inputs.upload-artifacts == 'true'
232+
with:
233+
name: benchmarkoor-${{ github.run_id }}
234+
path: ${{ runner.temp }}/benchmarkoor-results.tar.gz
235+
236+
- name: Cleanup
237+
shell: bash
238+
if: always()
239+
run: |
240+
echo "Cleaning up"
241+
rm -rf ${{ runner.temp }}/results
242+
rm -f ${{ runner.temp }}/benchmarkoor-config-url.yaml
243+
rm -f ${{ runner.temp }}/benchmarkoor-config-inline.yaml
244+
rm -f ${{ runner.temp }}/benchmarkoor-results.tar.gz
245+
docker rm -f benchmarkoor-run 2>/dev/null || true
246+
docker rm -f benchmarkoor-md 2>/dev/null || true

‎cmd/benchmarkoor/cleanup.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,9 @@ func performCleanup(ctx context.Context, dockerMgr docker.Manager, force bool) e
192192

193193
// Restore CPU frequency settings from orphaned state files and remove them.
194194
if len(cpufreqStateFiles) > 0 {
195-
if err := cpufreq.CleanupOrphanedCPUFreqState(ctx, log, cpufreqStateFiles); err != nil {
195+
if err := cpufreq.CleanupOrphanedCPUFreqState(
196+
ctx, log, cpufreqStateFiles, cpufreq.DefaultSysfsCPUPath,
197+
); err != nil {
196198
log.WithError(err).Warn("Failed to cleanup CPU frequency state files")
197199
}
198200
}

‎cmd/benchmarkoor/run.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ func runBenchmark(cmd *cobra.Command, args []string) error {
175175
}
176176
}
177177

178-
cpufreqMgr = cpufreq.NewManager(log, cacheDir)
178+
cpufreqMgr = cpufreq.NewManager(log, cacheDir, cfg.GetCPUSysfsPath())
179179
if err := cpufreqMgr.Start(ctx); err != nil {
180180
return fmt.Errorf("starting cpufreq manager: %w", err)
181181
}

‎config.example.docker.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ global:
33
client_logs_to_stdout: true
44
docker_network: benchmarkoor
55
cleanup_on_start: false
6-
drop_caches_path: /host_drop_caches
76

87
benchmark:
98
results_dir: ./results

‎config.example.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ global:
1818
# Optional: Override path to drop_caches file (default: /proc/sys/vm/drop_caches).
1919
# Useful when running in containers where the file is mounted at a different path.
2020
# drop_caches_path: /proc/sys/vm/drop_caches
21+
# Optional: Override sysfs base path for CPU frequency control (default: /sys/devices/system/cpu).
22+
# Useful when running in containers where /sys is read-only and the host path is bind-mounted
23+
# at a different location (e.g., -v /sys/devices/system/cpu:/host_sys_cpu).
24+
# cpu_sysfs_path: /sys/devices/system/cpu
2125
# Optional: GitHub token for downloading GitHub Actions artifacts via the REST API.
2226
# If the gh CLI is installed and authenticated, no token is needed.
2327
# Otherwise, provide a GitHub token with actions:read scope.

‎docker-compose.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ services:
88
# Required for dropping memory caches between tests (config.yaml#client.config.drop_memory_caches)
99
- SYS_ADMIN
1010
environment:
11+
- BENCHMARKOOR_GLOBAL_DROP_CACHES_PATH=/host_drop_caches
12+
- BENCHMARKOOR_GLOBAL_CPU_SYSFS_PATH=/host_sys_cpu
1113
- BENCHMARKOOR_GLOBAL_DIRECTORIES_TMP_DATADIR=${PWD}/tmp/data
1214
- BENCHMARKOOR_GLOBAL_DIRECTORIES_TMP_CACHEDIR=${PWD}/tmp/cache
1315
- BENCHMARKOOR_BENCHMARK_RESULTS_OWNER=${USER_UID}:${USER_GID}
@@ -18,6 +20,8 @@ services:
1820
- /sys/fs/cgroup:/sys/fs/cgroup:ro
1921
# Required for dropping memory caches between tests (config.yaml#client.config.drop_memory_caches)
2022
- /proc/sys/vm/drop_caches:/host_drop_caches
23+
# Required for CPU frequency control (config.yaml#client.config.resource_limits.cpu_freq)
24+
- /sys/devices/system/cpu:/host_sys_cpu
2125
# Config file
2226
- ./${BENCHMARKOOR_CONFIG:-config.example.docker.yaml}:/app/config.yaml:ro
2327
# Temp directory (For fetching temporary files and node data dirs)

‎docs/configuration.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ global:
8080
tmp_datadir: /tmp/benchmarkoor
8181
tmp_cachedir: /tmp/benchmarkoor-cache
8282
drop_caches_path: /proc/sys/vm/drop_caches
83+
cpu_sysfs_path: /sys/devices/system/cpu
8384
```
8485

8586
### Options
@@ -93,6 +94,7 @@ global:
9394
| `directories.tmp_datadir` | string | system temp | Directory for temporary datadir copies |
9495
| `directories.tmp_cachedir` | string | `~/.cache/benchmarkoor` | Directory for executor cache (git clones, etc.) |
9596
| `drop_caches_path` | string | `/proc/sys/vm/drop_caches` | Path to Linux drop_caches file (for containerized environments) |
97+
| `cpu_sysfs_path` | string | `/sys/devices/system/cpu` | Base path for CPU sysfs files (for containerized environments where `/sys` is read-only and the host path is bind-mounted elsewhere, e.g., `/host_sys_cpu`) |
9698
| `github_token` | string | - | GitHub token for downloading Actions artifacts via REST API. Not needed if `gh` CLI is installed and authenticated. Requires `actions:read` scope. Can also be set via `BENCHMARKOOR_GLOBAL_GITHUB_TOKEN` env var |
9799

98100
## Benchmark Settings
@@ -667,6 +669,7 @@ CPU frequency settings allow you to lock CPUs to a specific frequency, control t
667669
- Linux only
668670
- Root access (requires write access to `/sys/devices/system/cpu/*/cpufreq/`)
669671
- cpufreq subsystem must be available
672+
- When running in Docker, bind-mount `/sys/devices/system/cpu` into the container and set `global.cpu_sysfs_path` to the mount point (e.g., `/host_sys_cpu`)
670673

671674
```yaml
672675
resource_limits:

0 commit comments

Comments
 (0)