Skip to content

Commit da4b3b4

Browse files
authored
feat: allow select benchmarks to have exclusive machine use (#20903)
Still see odd spikes on some of these short verification tests. Maybe they're suffering from mem contention? Try this approach of letting them have exclusive machine access via the `PARALLEL=0` harness flag. Also slightly simplify parallelize scripts to not do denoising but make that a caller decision.
2 parents db496a4 + 38ae641 commit da4b3b4

6 files changed

Lines changed: 85 additions & 77 deletions

File tree

bootstrap.sh

Lines changed: 5 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export MAKEFLAGS="-j${MAKE_JOBS:-$(get_num_cpus)}"
2020
# We append all test commands to this file as they become available during build.
2121
# The test engine feeds it into parallel.
2222
export test_cmds_file="/tmp/test_cmds"
23+
export bench_cmds_file="/tmp/bench_cmds"
2324

2425
# Cleanup function. Called on script exit.
2526
function cleanup {
@@ -259,7 +260,6 @@ function stop_txes {
259260
function prep {
260261
check_toolchains
261262
pull_submodules
262-
rm -f $test_cmds_file
263263
}
264264

265265
function build {
@@ -346,61 +346,17 @@ function bench_merge {
346346

347347
}
348348

349-
function isolate_bench_cpus {
350-
[ "${CI:-0}" -eq 0 ] && return
351-
352-
# CPU layout assumption: physical cores are 0..N/2-1, hyperthreads are N/2..N-1.
353-
local total_cpus=$(nproc)
354-
local total_physical=$((total_cpus / 2))
355-
local os_reserve=8
356-
local bench_count=$((total_physical - os_reserve))
357-
358-
# Disable hyperthread siblings of benchmark cores (N/2 .. N/2+bench_count-1).
359-
# OS cores' hyperthreads (N/2+bench_count .. N-1) stay on for extra OS capacity.
360-
for cpu in $(seq $total_physical $((total_physical + bench_count - 1))); do
361-
sudo sh -c "echo 0 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
362-
done
363-
364-
# Pin all container processes to OS CPUs so they can't land on benchmark cores.
365-
# exec_test's taskset overrides this for each benchmark with its allocated CPUs.
366-
local os_cpu_list="$bench_count-$((total_physical - 1)),$((total_physical + bench_count))-$((total_cpus - 1))"
367-
echo "Pinning container processes to OS CPUs ($os_cpu_list)..."
368-
for pid in $(ps -eo pid= 2>/dev/null); do
369-
taskset -apc "$os_cpu_list" $pid &>/dev/null || true
370-
done
371-
372-
export BENCH_CPU_COUNT=$bench_count
373-
374-
echo "Benchmark CPU isolation: CPUs 0-$((bench_count - 1)) ($bench_count cores, hyperthreads off) for benchmarks."
375-
echo "OS CPUs: $os_cpu_list."
376-
}
377-
378-
function unisolate_bench_cpus {
379-
[ "${CI:-0}" -eq 0 ] && return
380-
381-
echo "Re-enabling all CPUs..."
382-
local total_cpus=$(nproc --all)
383-
for cpu in $(seq 1 $((total_cpus - 1))); do
384-
sudo sh -c "echo 1 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
385-
done
386-
# Unpin all processes (were pinned to OS CPUs during bench).
387-
for pid in $(ps -eo pid= 2>/dev/null); do
388-
taskset -apc 0-$((total_cpus - 1)) $pid &>/dev/null || true
389-
done
390-
echo "All CPUs re-enabled. Online CPUs: $(nproc)"
391-
}
392-
393349
function bench {
394350
# TODO bench for arm64.
395351
if [ $(arch) == arm64 ]; then
396352
return
397353
fi
398354
echo_header "bench all"
399355
build_bench
400-
isolate_bench_cpus
401-
find . -type d -iname bench-out | xargs rm -rf
402-
bench_cmds | STRICT_SCHEDULING=1 parallelize
403-
unisolate_bench_cpus
356+
357+
bench_cmds > $bench_cmds_file
358+
denoise "bench_engine $bench_cmds_file"
359+
404360
rm -rf bench-out
405361
mkdir -p bench-out
406362
bench_merge

ci3/bench_engine

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
# Uses strict scheduling to run benchmarks in parallel on their own cpus.
3+
# For benchmarks that can't be parallelized, runs them one at a time to avoid resource contention.
4+
# Isolates benchmark CPUs from OS and pins all other processes to non-bench CPUs to avoid interference.
5+
NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source
6+
7+
bench_cmds_file=$1
8+
9+
function isolate_bench_cpus {
10+
[ "$CI" -eq 0 ] && return
11+
12+
# CPU layout assumption: physical cores are 0..N/2-1, hyperthreads are N/2..N-1.
13+
local total_cpus=$(nproc)
14+
local total_physical=$((total_cpus / 2))
15+
local os_reserve=8
16+
local bench_count=$((total_physical - os_reserve))
17+
18+
# Disable hyperthread siblings of benchmark cores (N/2 .. N/2+bench_count-1).
19+
# OS cores' hyperthreads (N/2+bench_count .. N-1) stay on for extra OS capacity.
20+
for cpu in $(seq $total_physical $((total_physical + bench_count - 1))); do
21+
sudo sh -c "echo 0 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
22+
done
23+
24+
# Pin all container processes to OS CPUs so they can't land on benchmark cores.
25+
# exec_test's taskset overrides this for each benchmark with its allocated CPUs.
26+
local os_cpu_list="$bench_count-$((total_physical - 1)),$((total_physical + bench_count))-$((total_cpus - 1))"
27+
echo "Pinning container processes to OS CPUs ($os_cpu_list)..."
28+
for pid in $(ps -eo pid= 2>/dev/null); do
29+
taskset -apc "$os_cpu_list" $pid &>/dev/null || true
30+
done
31+
32+
export BENCH_CPU_COUNT=$bench_count
33+
34+
echo "Benchmark CPU isolation: CPUs 0-$((bench_count - 1)) ($bench_count cores, hyperthreads off) for benchmarks."
35+
echo "OS CPUs: $os_cpu_list."
36+
}
37+
38+
function unisolate_bench_cpus {
39+
[ "$CI" -eq 0 ] && return
40+
41+
echo "Re-enabling all CPUs..."
42+
local total_cpus=$(nproc --all)
43+
for cpu in $(seq 1 $((total_cpus - 1))); do
44+
sudo sh -c "echo 1 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
45+
done
46+
# Unpin all processes (were pinned to OS CPUs during bench).
47+
for pid in $(ps -eo pid= 2>/dev/null); do
48+
taskset -apc 0-$((total_cpus - 1)) $pid &>/dev/null || true
49+
done
50+
echo "All CPUs re-enabled. Online CPUs: $(nproc)"
51+
}
52+
53+
isolate_bench_cpus
54+
55+
# Clean up old benchmark outputs to avoid confusion with new results.
56+
find . -type d -iname bench-out | xargs rm -rf
57+
58+
# Run parallelizable benchmarks.
59+
cat $bench_cmds_file | grep -v ':PARALLEL=0' | STRICT_SCHEDULING=1 parallelize
60+
61+
# Run serial benchmarks one at a time (for memory bandwidth / cache isolation).
62+
serial_cmds=$(cat $bench_cmds_file | grep ':PARALLEL=0' || true)
63+
if [ -n "$serial_cmds" ]; then
64+
echo "Running serial benchmarks..."
65+
while IFS= read -r cmd; do
66+
[ -z "$cmd" ] && continue
67+
run_test_cmd "$cmd"
68+
done <<< "$serial_cmds"
69+
fi
70+
71+
unisolate_bench_cpus

ci3/parallelize

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ echo "Starting parallel run with max $jobs jobs..."
2626
export FLAKES_FILE="$root/.logged_test_flakes"
2727
echo "" > "$FLAKES_FILE" # Create/clear from previous run
2828

29-
# If we're in a terminal default to a progress bar, and use cache_log to save output to redis.
30-
# Otherwise use denoise to display dots and save output to redis.
3129
if [ -t 1 ]; then
30+
# If we're in a terminal default to a progress bar, and use cache_log to save output to redis.
31+
# Dump output only on failure.
3232
set +e
3333
output=$(parallel $parallel_args --bar 'DUMP_FAIL=1 run_test_cmd {}' > >(DUP=1 stdbuf -oL cache_log "Test run"))
3434
code=$?
@@ -38,7 +38,9 @@ if [ -t 1 ]; then
3838
fi
3939
set -e
4040
else
41-
denoise "parallel $parallel_args 'run_test_cmd {}'"
41+
# Non interactive environments (like CI) should just stream output.
42+
# Caller can denoise if they want to.
43+
parallel $parallel_args 'run_test_cmd {}'
4244
fi
4345

4446
slow_jobs=$(cat joblog.txt | \

ci3/parallelize_strict

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,6 @@ else
1414
num_cpus=$(get_num_cpus_max ${1:-})
1515
fi
1616

17-
[ -t 1 ] && no_term=0 || no_term=1
18-
1917
echo "Starting parallel run using max $num_cpus cpus..."
2018

2119
sem_sched init $num_cpus
@@ -26,8 +24,6 @@ function cleanup {
2624
}
2725
trap cleanup EXIT
2826

29-
trap 'echo >&2; exit' SIGINT SIGTERM
30-
3127
output=$(mktemp)
3228

3329
function run_and_release {
@@ -37,31 +33,17 @@ function run_and_release {
3733
run_test_cmd "$cmd" &
3834
local pid=$!
3935
wait $pid
40-
semc --name "/complete_tests" +1
41-
wait $pid
42-
}
43-
44-
function status {
45-
[ "$no_term" -eq 1 ] && return
46-
local complete_jobs=$(semc --name "/complete_tests")
47-
echo_stderr -e -n "\r$num_jobs scheduled, $(jobs -p | wc -l) running, $complete_jobs complete.\e[K"
4836
}
4937

5038
function wait_for_job {
5139
wait -n
5240
ret=$?
53-
status
5441
if [ $ret -ne 0 ]; then
55-
if [ $no_term -eq 0 ]; then
56-
echo_stderr
57-
cat $output | grep FAILED >&2
58-
fi
5942
exit 1
6043
fi
6144
}
6245

6346
function run_tests {
64-
semc --name "/complete_tests" --init 0
6547
while IFS= read -r cmd || [ -n "$cmd" ]; do
6648
# Default.
6749
local CPUS=2
@@ -86,23 +68,20 @@ function run_tests {
8668
run_and_release $CPU_LIST &
8769

8870
num_jobs=$((num_jobs + 1))
89-
status
9071
done
9172
}
9273

9374
num_jobs=0
9475

9576
# We'll handle errors explicitly.
9677
set +e
97-
run_tests > >(tee $output | cache_log "Test run")
78+
run_tests | tee $output
9879

9980
# Loop to monitor jobs until one fails or all finish.
10081
while (( $(jobs -p | wc -l) > 0 )); do
10182
wait_for_job
10283
done
10384

104-
[ -t 1 ] && echo
105-
10685
function filter_long_times {
10786
grep -E '\([0-9]+s\)$' | # Match lines ending with (number)s
10887
sed 's/.*(\([0-9]\+\)s)$/\1 &/' | # Extract number and keep original line

ci3/test_engine

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@ test_cmds_file=$1
1212
# Empty commands fed to run_test_cmd are no-ops, so we keep parallel processing results in timely fashion with this.
1313
while ! grep -Eq '^STOP$' $test_cmds_file; do sleep 5; echo | atomic_append $test_cmds_file; done &>/dev/null &
1414
# Continuously stream the test cmds into parallelize.
15-
DENOISE=0 parallelize < <(tail -n+0 -f $test_cmds_file) 2>/dev/null
15+
parallelize < <(tail -n+0 -f $test_cmds_file) 2>/dev/null

yarn-project/end-to-end/bootstrap.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ function bench_cmds {
117117
done
118118
echo "$hash:ISOLATE=1:NET=1:CPUS=8 barretenberg/cpp/scripts/ci_benchmark_browser_memory.sh ../../yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc"
119119

120-
# UltraHonk circuit benchmarks at different CPU counts
120+
# UltraHonk circuit benchmarks at different CPU counts (run serially for cache/bandwidth isolation)
121121
for cpus in 8 16 32; do
122-
echo "$hash:CPUS=$cpus barretenberg/cpp/scripts/ci_benchmark_ultrahonk_circuits.sh parity_base ../../yarn-project/end-to-end/$ultrahonk_bench_dir $cpus"
122+
echo "$hash:CPUS=$cpus:PARALLEL=0 barretenberg/cpp/scripts/ci_benchmark_ultrahonk_circuits.sh parity_base ../../yarn-project/end-to-end/$ultrahonk_bench_dir $cpus"
123123
done
124124
}
125125

0 commit comments

Comments
 (0)