feat: allow select benchmarks to have exclusive machine use

Still see odd spikes on some of these short verification tests. Maybe they're suffering from mem contention?
Try this approach of letting them have exclusive machine access via the `PARALLEL=0` harness flag.

Also slightly simplify parallelize scripts to not do denoising but make that a caller decision.

Author: charlielye

bootstrap.sh

@@ -20,6 +20,7 @@ export MAKEFLAGS="-j${MAKE_JOBS:-$(get_num_cpus)}"
 # We append all test commands to this file as they become available during build.
 # The test engine feeds it into parallel.
 export test_cmds_file="/tmp/test_cmds"
+export bench_cmds_file="/tmp/bench_cmds"
 
 # Cleanup function. Called on script exit.
 function cleanup {
@@ -259,7 +260,6 @@ function stop_txes {
 function prep {
   check_toolchains
   pull_submodules
-  rm -f $test_cmds_file
 }
 
 function build {
@@ -346,61 +346,17 @@ function bench_merge {
 
 }
 
-function isolate_bench_cpus {
-  [ "${CI:-0}" -eq 0 ] && return
-
-  # CPU layout assumption: physical cores are 0..N/2-1, hyperthreads are N/2..N-1.
-  local total_cpus=$(nproc)
-  local total_physical=$((total_cpus / 2))
-  local os_reserve=8
-  local bench_count=$((total_physical - os_reserve))
-
-  # Disable hyperthread siblings of benchmark cores (N/2 .. N/2+bench_count-1).
-  # OS cores' hyperthreads (N/2+bench_count .. N-1) stay on for extra OS capacity.
-  for cpu in $(seq $total_physical $((total_physical + bench_count - 1))); do
-    sudo sh -c "echo 0 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
-  done
-
-  # Pin all container processes to OS CPUs so they can't land on benchmark cores.
-  # exec_test's taskset overrides this for each benchmark with its allocated CPUs.
-  local os_cpu_list="$bench_count-$((total_physical - 1)),$((total_physical + bench_count))-$((total_cpus - 1))"
-  echo "Pinning container processes to OS CPUs ($os_cpu_list)..."
-  for pid in $(ps -eo pid= 2>/dev/null); do
-    taskset -apc "$os_cpu_list" $pid &>/dev/null || true
-  done
-
-  export BENCH_CPU_COUNT=$bench_count
-
-  echo "Benchmark CPU isolation: CPUs 0-$((bench_count - 1)) ($bench_count cores, hyperthreads off) for benchmarks."
-  echo "OS CPUs: $os_cpu_list."
-}
-
-function unisolate_bench_cpus {
-  [ "${CI:-0}" -eq 0 ] && return
-
-  echo "Re-enabling all CPUs..."
-  local total_cpus=$(nproc --all)
-  for cpu in $(seq 1 $((total_cpus - 1))); do
-    sudo sh -c "echo 1 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
-  done
-  # Unpin all processes (were pinned to OS CPUs during bench).
-  for pid in $(ps -eo pid= 2>/dev/null); do
-    taskset -apc 0-$((total_cpus - 1)) $pid &>/dev/null || true
-  done
-  echo "All CPUs re-enabled. Online CPUs: $(nproc)"
-}
-
 function bench {
   # TODO bench for arm64.
   if [ $(arch) == arm64 ]; then
     return
   fi
   echo_header "bench all"
   build_bench
-  isolate_bench_cpus
-  find . -type d -iname bench-out | xargs rm -rf
-  bench_cmds | STRICT_SCHEDULING=1 parallelize
-  unisolate_bench_cpus
+
+  bench_cmds > $bench_cmds_file
+  denoise "bench_engine $bench_cmds_file"
+
   rm -rf bench-out
   mkdir -p bench-out
   bench_merge

ci3/bench_engine

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Uses strict scheduling to run benchmarks in parallel on their own cpus.
+# For benchmarks that can't be parallelized, runs them one at a time to avoid resource contention.
+# Isolates benchmark CPUs from OS and pins all other processes to non-bench CPUs to avoid interference.
+NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source
+
+bench_cmds_file=$1
+
+function isolate_bench_cpus {
+  [ "$CI" -eq 0 ] && return
+
+  # CPU layout assumption: physical cores are 0..N/2-1, hyperthreads are N/2..N-1.
+  local total_cpus=$(nproc)
+  local total_physical=$((total_cpus / 2))
+  local os_reserve=8
+  local bench_count=$((total_physical - os_reserve))
+
+  # Disable hyperthread siblings of benchmark cores (N/2 .. N/2+bench_count-1).
+  # OS cores' hyperthreads (N/2+bench_count .. N-1) stay on for extra OS capacity.
+  for cpu in $(seq $total_physical $((total_physical + bench_count - 1))); do
+    sudo sh -c "echo 0 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
+  done
+
+  # Pin all container processes to OS CPUs so they can't land on benchmark cores.
+  # exec_test's taskset overrides this for each benchmark with its allocated CPUs.
+  local os_cpu_list="$bench_count-$((total_physical - 1)),$((total_physical + bench_count))-$((total_cpus - 1))"
+  echo "Pinning container processes to OS CPUs ($os_cpu_list)..."
+  for pid in $(ps -eo pid= 2>/dev/null); do
+    taskset -apc "$os_cpu_list" $pid &>/dev/null || true
+  done
+
+  export BENCH_CPU_COUNT=$bench_count
+
+  echo "Benchmark CPU isolation: CPUs 0-$((bench_count - 1)) ($bench_count cores, hyperthreads off) for benchmarks."
+  echo "OS CPUs: $os_cpu_list."
+}
+
+function unisolate_bench_cpus {
+  [ "$CI" -eq 0 ] && return
+
+  echo "Re-enabling all CPUs..."
+  local total_cpus=$(nproc --all)
+  for cpu in $(seq 1 $((total_cpus - 1))); do
+    sudo sh -c "echo 1 > /sys/devices/system/cpu/cpu$cpu/online" 2>/dev/null || true
+  done
+  # Unpin all processes (were pinned to OS CPUs during bench).
+  for pid in $(ps -eo pid= 2>/dev/null); do
+    taskset -apc 0-$((total_cpus - 1)) $pid &>/dev/null || true
+  done
+  echo "All CPUs re-enabled. Online CPUs: $(nproc)"
+}
+
+isolate_bench_cpus
+
+# Clean up old benchmark outputs to avoid confusion with new results.
+find . -type d -iname bench-out | xargs rm -rf
+
+# Run parallelizable benchmarks.
+cat $bench_cmds_file | grep -v ':PARALLEL=0' | STRICT_SCHEDULING=1 parallelize
+
+# Run serial benchmarks one at a time (for memory bandwidth / cache isolation).
+serial_cmds=$(cat $bench_cmds_file | grep ':PARALLEL=0' || true)
+if [ -n "$serial_cmds" ]; then
+  echo "Running serial benchmarks..."
+  while IFS= read -r cmd; do
+    [ -z "$cmd" ] && continue
+    run_test_cmd "$cmd"
+  done <<< "$serial_cmds"
+fi
+
+unisolate_bench_cpus

ci3/parallelize

@@ -26,9 +26,9 @@ echo "Starting parallel run with max $jobs jobs..."
 export FLAKES_FILE="$root/.logged_test_flakes"
 echo "" > "$FLAKES_FILE"  # Create/clear from previous run
 
-# If we're in a terminal default to a progress bar, and use cache_log to save output to redis.
-# Otherwise use denoise to display dots and save output to redis.
 if [ -t 1 ]; then
+  # If we're in a terminal default to a progress bar, and use cache_log to save output to redis.
+  # Dump output only on failure.
   set +e
   output=$(parallel $parallel_args --bar 'DUMP_FAIL=1 run_test_cmd {}' > >(DUP=1 stdbuf -oL cache_log "Test run"))
   code=$?
@@ -38,7 +38,9 @@ if [ -t 1 ]; then
   fi
   set -e
 else
-  denoise "parallel $parallel_args 'run_test_cmd {}'"
+  # Non interactive environments (like CI) should just stream output.
+  # Caller can denoise if they want to.
+  parallel $parallel_args 'run_test_cmd {}'
 fi
 
 slow_jobs=$(cat joblog.txt | \

ci3/parallelize_strict

@@ -14,8 +14,6 @@ else
   num_cpus=$(get_num_cpus_max ${1:-})
 fi
 
-[ -t 1 ] && no_term=0 || no_term=1
-
 echo "Starting parallel run using max $num_cpus cpus..."
 
 sem_sched init $num_cpus
@@ -26,8 +24,6 @@ function cleanup {
 }
 trap cleanup EXIT
 
-trap 'echo >&2; exit' SIGINT SIGTERM
-
 output=$(mktemp)
 
 function run_and_release {
@@ -37,31 +33,17 @@ function run_and_release {
   run_test_cmd "$cmd" &
   local pid=$!
   wait $pid
-  semc --name "/complete_tests" +1
-  wait $pid
-}
-
-function status {
-  [ "$no_term" -eq 1 ] && return
-  local complete_jobs=$(semc --name "/complete_tests")
-  echo_stderr -e -n "\r$num_jobs scheduled, $(jobs -p | wc -l) running, $complete_jobs complete.\e[K"
 }
 
 function wait_for_job {
   wait -n
   ret=$?
-  status
   if [ $ret -ne 0 ]; then
-    if [ $no_term -eq 0 ]; then
-      echo_stderr
-      cat $output | grep FAILED >&2
-    fi
     exit 1
   fi
 }
 
 function run_tests {
-  semc --name "/complete_tests" --init 0
   while IFS= read -r cmd || [ -n "$cmd" ]; do
     # Default.
     local CPUS=2
@@ -86,23 +68,20 @@ function run_tests {
     run_and_release $CPU_LIST &
 
     num_jobs=$((num_jobs + 1))
-    status
   done
 }
 
 num_jobs=0
 
 # We'll handle errors explicitly.
 set +e
-run_tests > >(tee $output | cache_log "Test run")
+run_tests | tee $output
 
 # Loop to monitor jobs until one fails or all finish.
 while (( $(jobs -p | wc -l) > 0 )); do
   wait_for_job
 done
 
-[ -t 1 ] && echo
-
 function filter_long_times {
   grep -E '\([0-9]+s\)$' |           # Match lines ending with (number)s
   sed 's/.*(\([0-9]\+\)s)$/\1 &/' |  # Extract number and keep original line

ci3/test_engine

@@ -12,4 +12,4 @@ test_cmds_file=$1
 # Empty commands fed to run_test_cmd are no-ops, so we keep parallel processing results in timely fashion with this.
 while ! grep -Eq '^STOP$' $test_cmds_file; do sleep 5; echo | atomic_append $test_cmds_file; done &>/dev/null &
 # Continuously stream the test cmds into parallelize.
-DENOISE=0 parallelize < <(tail -n+0 -f $test_cmds_file) 2>/dev/null
+parallelize < <(tail -n+0 -f $test_cmds_file) 2>/dev/null

yarn-project/end-to-end/bootstrap.sh

@@ -117,9 +117,9 @@ function bench_cmds {
   done
   echo "$hash:ISOLATE=1:NET=1:CPUS=8 barretenberg/cpp/scripts/ci_benchmark_browser_memory.sh ../../yarn-project/end-to-end/example-app-ivc-inputs-out/ecdsar1+transfer_0_recursions+sponsored_fpc"
 
-  # UltraHonk circuit benchmarks at different CPU counts
+  # UltraHonk circuit benchmarks at different CPU counts (run serially for cache/bandwidth isolation)
   for cpus in 8 16 32; do
-    echo "$hash:CPUS=$cpus barretenberg/cpp/scripts/ci_benchmark_ultrahonk_circuits.sh parity_base ../../yarn-project/end-to-end/$ultrahonk_bench_dir $cpus"
+    echo "$hash:CPUS=$cpus:PARALLEL=0 barretenberg/cpp/scripts/ci_benchmark_ultrahonk_circuits.sh parity_base ../../yarn-project/end-to-end/$ultrahonk_bench_dir $cpus"
   done
 }