From 1c14437fd8dec43c52460dec2c4d7573ebcb439d Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Wed, 22 Apr 2026 11:24:07 +0000
Subject: [PATCH 01/11] debug(kv-store): instrument test:browser to diagnose CI
 hang

Temporary diagnostic wrapper around `yarn test:browser` that runs
with a 90s timeout and captures system state (cgroup cpu.stat, memory,
/tmp usage, process states) every 1s to /tmp/diag/probe.log, then
dumps the tail to stderr on exit.

Goal: diagnose the CI hang documented in PR #22693 where the kv-store
browser tests hang silently around test file 5 only in the CI ISOLATE
container, not reproducible from the same docker args locally.

Revert once root-caused.
---
 yarn-project/kv-store/package.json            |  2 +-
 yarn-project/kv-store/package.local.json      |  2 +-
 .../kv-store/scripts/probe-test-browser.sh    | 61 +++++++++++++++++++
 3 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100755 yarn-project/kv-store/scripts/probe-test-browser.sh

diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json
index 0bd58a4d3cde..e38c3a9dc01d 100644
--- a/yarn-project/kv-store/package.json
+++ b/yarn-project/kv-store/package.json
@@ -18,7 +18,7 @@
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "test:browser": "vitest run --config ./vitest.config.ts",
     "bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
-    "test": "yarn test:node && yarn test:browser",
+    "test": "yarn test:node && bash scripts/probe-test-browser.sh",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "inherits": [
diff --git a/yarn-project/kv-store/package.local.json b/yarn-project/kv-store/package.local.json
index d93348f53280..093a5e2e5e94 100644
--- a/yarn-project/kv-store/package.local.json
+++ b/yarn-project/kv-store/package.local.json
@@ -1,5 +1,5 @@
 {
   "scripts": {
-    "test": "yarn test:node && yarn test:browser"
+    "test": "yarn test:node && bash scripts/probe-test-browser.sh"
   }
 }
diff --git a/yarn-project/kv-store/scripts/probe-test-browser.sh b/yarn-project/kv-store/scripts/probe-test-browser.sh
new file mode 100755
index 000000000000..30ff73b6f327
--- /dev/null
+++ b/yarn-project/kv-store/scripts/probe-test-browser.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Temporary diagnostic wrapper for `yarn test:browser` used to capture
+# CI container state around the intermittent hang documented in PR #22693.
+# Revert this and the package.local.json change once root-caused.
+
+cd "$(dirname "$0")/.."
+
+PROBE_LOG=/tmp/diag/probe.log
+mkdir -p "$(dirname "$PROBE_LOG")"
+: > "$PROBE_LOG"
+
+echo "probe-test-browser: starting at $(date +%T) (pid $$)" >&2
+
+probe_loop() {
+  while true; do
+    {
+      echo "=== $(date +%T.%N | cut -c1-12) ==="
+      echo "-- /tmp --"
+      df -h /tmp 2>/dev/null | tail -1
+      du -sh /tmp 2>/dev/null
+      echo "-- cgroup mem --"
+      if [ -r /sys/fs/cgroup/memory.current ]; then
+        awk '{printf "current %d bytes (%.0f MB)\n", $1, $1/1048576}' /sys/fs/cgroup/memory.current
+        cat /sys/fs/cgroup/memory.events 2>/dev/null
+      fi
+      echo "-- cgroup cpu --"
+      cat /sys/fs/cgroup/cpu.stat 2>/dev/null
+      echo "-- top by rss --"
+      ps -eo pid,ppid,stat,rss,pcpu,wchan:20,comm --sort=-rss --no-headers 2>/dev/null | head -12
+      echo "-- chromium procs --"
+      for pid in $(pgrep -f 'chrome|chromium' 2>/dev/null); do
+        state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null)
+        wchan=$(cat "/proc/$pid/wchan" 2>/dev/null)
+        rss_kb=$(awk '/^VmRSS:/{print $2}' "/proc/$pid/status" 2>/dev/null)
+        threads=$(awk '/^Threads:/{print $2}' "/proc/$pid/status" 2>/dev/null)
+        cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-100)
+        echo "pid=$pid state=$state threads=$threads rss=${rss_kb}kB wchan=$wchan cmd=$cmd"
+      done
+    } >> "$PROBE_LOG" 2>&1
+    sleep 1
+  done
+}
+
+probe_loop &
+PROBE_PID=$!
+
+cleanup() {
+  rc=$?
+  kill "$PROBE_PID" 2>/dev/null
+  wait "$PROBE_PID" 2>/dev/null
+  {
+    echo ""
+    echo "=== PROBE LOG (last 400 lines, test exit=$rc) ==="
+    tail -400 "$PROBE_LOG"
+    echo "=== END PROBE LOG ==="
+  } >&2
+  exit "$rc"
+}
+trap cleanup EXIT
+
+timeout -v 90s yarn test:browser

From 63e2b912b2e0c4cbd65102d796fb2217b758d023 Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Wed, 22 Apr 2026 13:24:57 +0000
Subject: [PATCH 02/11] debug(kv-store): v2 probe with hybrid cadence and
 kernel/socket signals

Upgrade the temporary CI diagnostic wrapper so the next hang captures
enough ground truth to identify the stall site.

- Dump full probe + stacks log on exit (v1 tail -400 truncated the
  transition moment).
- Hybrid cadence: 1s steady, 0.1s burst when `yarn test:browser` stdout
  has been silent for >3s, capped at ~20s dense sampling.
- Per-snapshot: /proc/net/{tcp,tcp6,unix} state counts.
- Per-burst stacks: lsof -i TCP -U, /proc/\$pid/syscall, per-thread wchan
  histogram, socket/pipe/anon_inode FD symlinks.
- Widen pids_of_interest to node|vitest|esbuild|yarn so we see the full
  vitest process tree, not just chromium.
---
 .../kv-store/scripts/probe-test-browser.sh    | 163 ++++++++++++++----
 1 file changed, 131 insertions(+), 32 deletions(-)

diff --git a/yarn-project/kv-store/scripts/probe-test-browser.sh b/yarn-project/kv-store/scripts/probe-test-browser.sh
index 30ff73b6f327..567aeefe95b8 100755
--- a/yarn-project/kv-store/scripts/probe-test-browser.sh
+++ b/yarn-project/kv-store/scripts/probe-test-browser.sh
@@ -1,43 +1,131 @@
 #!/usr/bin/env bash
 # Temporary diagnostic wrapper for `yarn test:browser` used to capture
 # CI container state around the intermittent hang documented in PR #22693.
+# v2: full log dump, hybrid 1s/0.1s cadence, kernel stacks, socket state.
 # Revert this and the package.local.json change once root-caused.
 
 cd "$(dirname "$0")/.."
 
-PROBE_LOG=/tmp/diag/probe.log
-mkdir -p "$(dirname "$PROBE_LOG")"
+DIAG_DIR=/tmp/diag
+PROBE_LOG=$DIAG_DIR/probe.log
+STACKS_LOG=$DIAG_DIR/stacks.log
+VITEST_LOG=$DIAG_DIR/vitest.log
+mkdir -p "$DIAG_DIR"
 : > "$PROBE_LOG"
+: > "$STACKS_LOG"
+: > "$VITEST_LOG"
 
-echo "probe-test-browser: starting at $(date +%T) (pid $$)" >&2
+echo "probe-test-browser v2: starting at $(date +%T) (pid $$)" >&2
+
+pids_of_interest() {
+  pgrep -f 'chrom|node|vitest|esbuild|yarn' 2>/dev/null
+}
+
+snapshot() {
+  local tag="$1"
+  echo "=== $(date +%T.%N | cut -c1-12) $tag ==="
+  echo "-- /tmp --"
+  df -h /tmp 2>/dev/null | tail -1
+  du -sh /tmp 2>/dev/null
+  echo "-- cgroup mem --"
+  if [ -r /sys/fs/cgroup/memory.current ]; then
+    awk '{printf "current %d bytes (%.0f MB)\n", $1, $1/1048576}' /sys/fs/cgroup/memory.current
+    cat /sys/fs/cgroup/memory.events 2>/dev/null
+  fi
+  echo "-- cgroup cpu --"
+  cat /sys/fs/cgroup/cpu.stat 2>/dev/null
+  echo "-- top by rss --"
+  ps -eo pid,ppid,stat,rss,pcpu,wchan:20,comm --sort=-rss --no-headers 2>/dev/null | head -15
+  echo "-- procs of interest --"
+  for pid in $(pids_of_interest); do
+    [ -d "/proc/$pid" ] || continue
+    local state threads rss_kb wchan cmd
+    state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null)
+    wchan=$(cat "/proc/$pid/wchan" 2>/dev/null)
+    rss_kb=$(awk '/^VmRSS:/{print $2}' "/proc/$pid/status" 2>/dev/null)
+    threads=$(awk '/^Threads:/{print $2}' "/proc/$pid/status" 2>/dev/null)
+    cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-120)
+    echo "pid=$pid state=$state threads=$threads rss=${rss_kb}kB wchan=$wchan cmd=$cmd"
+  done
+  # TCP state hex (col 4): 01=ESTABLISHED 04=FIN_WAIT1 05=FIN_WAIT2 06=TIME_WAIT
+  # 07=CLOSE 08=CLOSE_WAIT 09=LAST_ACK 0A=LISTEN 0B=CLOSING
+  echo "-- tcp4 sockets (state counts) --"
+  awk 'NR>1{print $4}' /proc/net/tcp 2>/dev/null | sort | uniq -c
+  echo "-- tcp6 sockets (state counts) --"
+  awk 'NR>1{print $4}' /proc/net/tcp6 2>/dev/null | sort | uniq -c
+  # Unix socket state (col 6): 01=SS_UNCONNECTED 02=SS_CONNECTING 03=SS_CONNECTED
+  # 04=SS_DISCONNECTING — CDP/vitest IPC is typically over SS_CONNECTED unix sockets.
+  echo "-- unix sockets (state counts) --"
+  awk 'NR>1{print $6}' /proc/net/unix 2>/dev/null | sort | uniq -c
+}
+
+stacks_snapshot() {
+  local tag="$1"
+  echo "=== $(date +%T.%N | cut -c1-12) $tag ==="
+  echo "-- lsof tcp/unix (-n -P, head -40) --"
+  lsof -n -P -i TCP 2>/dev/null | head -25
+  lsof -n -P -U 2>/dev/null | head -15
+  for pid in $(pids_of_interest); do
+    [ -d "/proc/$pid" ] || continue
+    echo "--- pid=$pid ---"
+    tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-140
+    echo ""
+    # /proc/$pid/syscall: syscall_nr arg0..arg5 sp pc (readable by process owner)
+    echo "syscall:"
+    cat "/proc/$pid/syscall" 2>/dev/null | head -1
+    # /proc/$pid/stack: may require CAP_SYS_ADMIN; print only if non-empty
+    local stack_content
+    stack_content=$(cat "/proc/$pid/stack" 2>/dev/null | head -15)
+    if [ -n "$stack_content" ]; then
+      echo "kernel stack:"
+      echo "$stack_content"
+    fi
+    # Per-thread wchan — tells us if threads are stuck on different things
+    echo "thread wchans (sorted | uniq -c):"
+    for t in /proc/$pid/task/*/wchan; do
+      cat "$t" 2>/dev/null
+      echo ""
+    done | sort | uniq -c | sort -rn | head -10
+    echo "socket/pipe fds:"
+    for fd in /proc/$pid/fd/*; do
+      local target
+      target=$(readlink "$fd" 2>/dev/null) || continue
+      case "$target" in
+        socket:*|pipe:*|anon_inode:*) echo "  $(basename "$fd") -> $target" ;;
+      esac
+    done
+  done
+}
 
 probe_loop() {
+  local last_vitest_size=0
+  local silent_since
+  silent_since=$(date +%s)
+  local burst_count=0
+  local burst_cap=200  # ~20s dense sampling at 0.1s
   while true; do
-    {
-      echo "=== $(date +%T.%N | cut -c1-12) ==="
-      echo "-- /tmp --"
-      df -h /tmp 2>/dev/null | tail -1
-      du -sh /tmp 2>/dev/null
-      echo "-- cgroup mem --"
-      if [ -r /sys/fs/cgroup/memory.current ]; then
-        awk '{printf "current %d bytes (%.0f MB)\n", $1, $1/1048576}' /sys/fs/cgroup/memory.current
-        cat /sys/fs/cgroup/memory.events 2>/dev/null
+    local now_s cur_size silent_for
+    now_s=$(date +%s)
+    cur_size=$(stat -c %s "$VITEST_LOG" 2>/dev/null || echo 0)
+    if [ "$cur_size" != "$last_vitest_size" ]; then
+      last_vitest_size=$cur_size
+      silent_since=$now_s
+      burst_count=0
+    fi
+    silent_for=$((now_s - silent_since))
+
+    if [ "$silent_for" -ge 3 ] && [ "$burst_count" -lt "$burst_cap" ]; then
+      snapshot "burst(${silent_for}s silent)" >> "$PROBE_LOG" 2>&1
+      # Stacks every 10th burst tick (~every 1s of burst) to keep log size sane
+      if [ $((burst_count % 10)) -eq 0 ]; then
+        stacks_snapshot "burst(${silent_for}s silent)" >> "$STACKS_LOG" 2>&1
       fi
-      echo "-- cgroup cpu --"
-      cat /sys/fs/cgroup/cpu.stat 2>/dev/null
-      echo "-- top by rss --"
-      ps -eo pid,ppid,stat,rss,pcpu,wchan:20,comm --sort=-rss --no-headers 2>/dev/null | head -12
-      echo "-- chromium procs --"
-      for pid in $(pgrep -f 'chrome|chromium' 2>/dev/null); do
-        state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null)
-        wchan=$(cat "/proc/$pid/wchan" 2>/dev/null)
-        rss_kb=$(awk '/^VmRSS:/{print $2}' "/proc/$pid/status" 2>/dev/null)
-        threads=$(awk '/^Threads:/{print $2}' "/proc/$pid/status" 2>/dev/null)
-        cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-100)
-        echo "pid=$pid state=$state threads=$threads rss=${rss_kb}kB wchan=$wchan cmd=$cmd"
-      done
-    } >> "$PROBE_LOG" 2>&1
-    sleep 1
+      burst_count=$((burst_count + 1))
+      sleep 0.1
+    else
+      snapshot "steady" >> "$PROBE_LOG" 2>&1
+      sleep 1
+    fi
   done
 }
 
@@ -45,17 +133,28 @@ probe_loop &
 PROBE_PID=$!
 
 cleanup() {
-  rc=$?
+  local rc=$?
   kill "$PROBE_PID" 2>/dev/null
   wait "$PROBE_PID" 2>/dev/null
+  # One last stacks snapshot at the moment of failure
+  stacks_snapshot "cleanup" >> "$STACKS_LOG" 2>&1
   {
     echo ""
-    echo "=== PROBE LOG (last 400 lines, test exit=$rc) ==="
-    tail -400 "$PROBE_LOG"
-    echo "=== END PROBE LOG ==="
+    echo "=== VITEST LOG (tail -300) ==="
+    tail -300 "$VITEST_LOG"
+    echo ""
+    echo "=== FULL PROBE LOG ($(wc -l < "$PROBE_LOG") lines) ==="
+    cat "$PROBE_LOG"
+    echo ""
+    echo "=== FULL STACKS LOG ($(wc -l < "$STACKS_LOG") lines, test exit=$rc) ==="
+    cat "$STACKS_LOG"
+    echo "=== END DIAGNOSTICS ==="
   } >&2
   exit "$rc"
 }
 trap cleanup EXIT
 
-timeout -v 90s yarn test:browser
+# Tee vitest output to a file so the probe loop can detect silence.
+# pipefail ensures we surface timeout's exit code, not tee's.
+set -o pipefail
+timeout -v 90s yarn test:browser 2>&1 | tee "$VITEST_LOG"

From 18c23ec1d8352313bd9efcb9c997b3220147a62f Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Wed, 22 Apr 2026 13:54:41 +0000
Subject: [PATCH 03/11] debug(kv-store): probe v2.1 - periodic in-flight stacks
 baseline

Two small tweaks after observing a passing CI run (6e65957a4521a43d):

- Capture stacks_snapshot every 10 steady iterations (~10s of wall time)
  so we have "healthy waiting" reference data to diff against
  burst-triggered snapshots during a future hang. Without this, a
  passing run produced zero in-flight stacks because the burst trigger
  never armed.
- Silence the benign "No such file or directory" race when /proc/\$pid/
  disappears mid-snapshot by moving the redirect around the read, not
  around the pipe (the pipe's output-side didn't inherit it).
---
 yarn-project/kv-store/scripts/probe-test-browser.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/yarn-project/kv-store/scripts/probe-test-browser.sh b/yarn-project/kv-store/scripts/probe-test-browser.sh
index 567aeefe95b8..8c23e566ba51 100755
--- a/yarn-project/kv-store/scripts/probe-test-browser.sh
+++ b/yarn-project/kv-store/scripts/probe-test-browser.sh
@@ -44,7 +44,7 @@ snapshot() {
     wchan=$(cat "/proc/$pid/wchan" 2>/dev/null)
     rss_kb=$(awk '/^VmRSS:/{print $2}' "/proc/$pid/status" 2>/dev/null)
     threads=$(awk '/^Threads:/{print $2}' "/proc/$pid/status" 2>/dev/null)
-    cmd=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-120)
+    cmd=$({ tr '\0' ' ' < "/proc/$pid/cmdline"; } 2>/dev/null | cut -c1-120)
     echo "pid=$pid state=$state threads=$threads rss=${rss_kb}kB wchan=$wchan cmd=$cmd"
   done
   # TCP state hex (col 4): 01=ESTABLISHED 04=FIN_WAIT1 05=FIN_WAIT2 06=TIME_WAIT
@@ -68,7 +68,7 @@ stacks_snapshot() {
   for pid in $(pids_of_interest); do
     [ -d "/proc/$pid" ] || continue
     echo "--- pid=$pid ---"
-    tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | cut -c1-140
+    { tr '\0' ' ' < "/proc/$pid/cmdline"; } 2>/dev/null | cut -c1-140
     echo ""
     # /proc/$pid/syscall: syscall_nr arg0..arg5 sp pc (readable by process owner)
     echo "syscall:"
@@ -102,7 +102,8 @@ probe_loop() {
   local silent_since
   silent_since=$(date +%s)
   local burst_count=0
-  local burst_cap=200  # ~20s dense sampling at 0.1s
+  local burst_cap=200   # ~20s dense sampling at 0.1s
+  local steady_count=0  # increments once per steady iteration
   while true; do
     local now_s cur_size silent_for
     now_s=$(date +%s)
@@ -124,6 +125,12 @@ probe_loop() {
       sleep 0.1
     else
       snapshot "steady" >> "$PROBE_LOG" 2>&1
+      # Periodic in-flight stacks baseline — gives us a "healthy waiting"
+      # reference to diff against burst-triggered dumps during a hang.
+      if [ $((steady_count % 10)) -eq 0 ]; then
+        stacks_snapshot "steady(tick=${steady_count})" >> "$STACKS_LOG" 2>&1
+      fi
+      steady_count=$((steady_count + 1))
       sleep 1
     fi
   done

From b0994b4213a2780a19e392d5219475bb1a9002dd Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Wed, 22 Apr 2026 14:31:58 +0000
Subject: [PATCH 04/11] debug(kv-store): probe v2.2 - CPU availability
 microbench

Force a fixed 100k-iteration awk arithmetic loop under bash's `time`
keyword each snapshot; emit real/user/sys as a "microbench:" line.

The (user+sys)/real ratio disambiguates the two scenarios consistent
with the v1 hang data (flat usage_usec, threads on futex_wait_queue):

  - scenario A (vitest/chromium IPC deadlock): no process in our cgroup
    is demanding CPU, so microbench gets all it asks for -> ratio ~1.0
  - scenario B (host-level preemption / noisy neighbor): microbench
    demands CPU but host scheduler doesn't serve it -> ratio <<1.0

Fires every 5th steady iteration (~0.4% overhead during tests) and
every burst iteration (overhead irrelevant during a hang since tests
aren't running anyway).
---
 .../kv-store/scripts/probe-test-browser.sh    | 27 +++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/yarn-project/kv-store/scripts/probe-test-browser.sh b/yarn-project/kv-store/scripts/probe-test-browser.sh
index 8c23e566ba51..0148594b4953 100755
--- a/yarn-project/kv-store/scripts/probe-test-browser.sh
+++ b/yarn-project/kv-store/scripts/probe-test-browser.sh
@@ -21,6 +21,16 @@ pids_of_interest() {
   pgrep -f 'chrom|node|vitest|esbuild|yarn' 2>/dev/null
 }
 
+# Force fixed CPU demand and measure what fraction we actually got.
+# ratio = (user+sys)/real. ~1.0 = full CPU; <0.5 = heavily preempted by host.
+# Disambiguates "processes not asking for CPU" (scenario A) from "host not
+# scheduling us" (scenario B) when cgroup counters are flat.
+microbench() {
+  local TIMEFORMAT='microbench: real=%R user=%U sys=%S'
+  { time awk 'BEGIN{x=0; for(i=0;i<100000;i++) x+=i*i; print x > "/dev/null"}' \
+    </dev/null 2>/dev/null; } 2>&1
+}
+
 snapshot() {
   local tag="$1"
   echo "=== $(date +%T.%N | cut -c1-12) $tag ==="
@@ -116,7 +126,13 @@ probe_loop() {
     silent_for=$((now_s - silent_since))
 
     if [ "$silent_for" -ge 3 ] && [ "$burst_count" -lt "$burst_cap" ]; then
-      snapshot "burst(${silent_for}s silent)" >> "$PROBE_LOG" 2>&1
+      {
+        snapshot "burst(${silent_for}s silent)"
+        # During a hang, microbench every burst snapshot — it's the core
+        # signal for "is the host giving us CPU?" Overhead is irrelevant
+        # since tests aren't running.
+        microbench
+      } >> "$PROBE_LOG" 2>&1
       # Stacks every 10th burst tick (~every 1s of burst) to keep log size sane
       if [ $((burst_count % 10)) -eq 0 ]; then
         stacks_snapshot "burst(${silent_for}s silent)" >> "$STACKS_LOG" 2>&1
@@ -124,7 +140,14 @@ probe_loop() {
       burst_count=$((burst_count + 1))
       sleep 0.1
     else
-      snapshot "steady" >> "$PROBE_LOG" 2>&1
+      {
+        snapshot "steady"
+        # Microbench every 5th steady iteration to bound CPU overhead
+        # while tests are actually running (~0.4% vs tests' CPU budget).
+        if [ $((steady_count % 5)) -eq 0 ]; then
+          microbench
+        fi
+      } >> "$PROBE_LOG" 2>&1
       # Periodic in-flight stacks baseline — gives us a "healthy waiting"
       # reference to diff against burst-triggered dumps during a hang.
       if [ $((steady_count % 10)) -eq 0 ]; then

From a337f119c5dacfca646b9a62d49a6de0f70000cf Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 10:50:10 +0000
Subject: [PATCH 05/11] fix(kv-store): isolate browser tests per file to avoid
 CDP teardown deadlock

Vitest+chromium hit a CDP CLOSE_WAIT deadlock at test-file transitions
under CPU-constrained environments (CI3 ISOLATE: --cpus=2). Vitest closes
6 of 10 CDP TCP connections in a teardown batch; chromium's network
service can't drain them under contention; vitest's teardown blocks
indefinitely on the close-handshake. Both processes end up with zero
on-CPU threads (vitest on futex_wait_queue, chromium on ep_poll). Outer
ci3 timeout fires after ~90s of silence, CI fails.

Fix: run each test file in a separate vitest invocation, so CDP teardown
only happens at process exit (where the OS reaps everything). Cost is
~5-10s of vite startup per file, ~60-100s extra per browser test run.

Also adds:
- scripts/repro-browser-hang.sh: reproduces the hang reliably under
  docker_isolate (--cpus=2 --memory=8g --tmpfs /tmp:exec,size=1g) for
  future regression checks.
- scripts/probe-test-browser.sh: deep diagnostic wrapper with hybrid
  steady/burst sampling of /proc state, cgroup counters, and socket
  state. Used to root-cause this bug; kept for future debugging.
- .test_patterns.yml: transfer kv-store TIMEOUT flake-entry ownership
  to martin (the original owner alex held it from a prior incident).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .test_patterns.yml                            |  2 +-
 yarn-project/kv-store/package.json            |  4 +-
 yarn-project/kv-store/package.local.json      |  3 +-
 .../kv-store/scripts/probe-test-browser.sh    | 27 +++++++++---
 .../kv-store/scripts/repro-browser-hang.sh    | 42 +++++++++++++++++++
 .../kv-store/scripts/run-browser-tests.sh     | 37 ++++++++++++++++
 6 files changed, 106 insertions(+), 9 deletions(-)
 create mode 100755 yarn-project/kv-store/scripts/repro-browser-hang.sh
 create mode 100755 yarn-project/kv-store/scripts/run-browser-tests.sh

diff --git a/.test_patterns.yml b/.test_patterns.yml
index 5a334cd586c6..79cdcb8f3a9a 100644
--- a/.test_patterns.yml
+++ b/.test_patterns.yml
@@ -153,7 +153,7 @@ tests:
   - regex: "yarn-project/kv-store"
     error_regex: "timeout: sending signal TERM to command"
     owners:
-      - *alex
+      - *martin
   - regex: "yarn-project/kv-store"
     error_regex: "Failed to fetch dynamically imported module"
     owners:
diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json
index e38c3a9dc01d..b9efae78907a 100644
--- a/yarn-project/kv-store/package.json
+++ b/yarn-project/kv-store/package.json
@@ -16,9 +16,9 @@
     "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
-    "test:browser": "vitest run --config ./vitest.config.ts",
+    "test:browser": "bash scripts/run-browser-tests.sh",
     "bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
-    "test": "yarn test:node && bash scripts/probe-test-browser.sh",
+    "test": "yarn test:node && yarn test:browser",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "inherits": [
diff --git a/yarn-project/kv-store/package.local.json b/yarn-project/kv-store/package.local.json
index 093a5e2e5e94..87f59219bf36 100644
--- a/yarn-project/kv-store/package.local.json
+++ b/yarn-project/kv-store/package.local.json
@@ -1,5 +1,6 @@
 {
   "scripts": {
-    "test": "yarn test:node && bash scripts/probe-test-browser.sh"
+    "test": "yarn test:node && yarn test:browser",
+    "test:browser": "bash scripts/run-browser-tests.sh"
   }
 }
diff --git a/yarn-project/kv-store/scripts/probe-test-browser.sh b/yarn-project/kv-store/scripts/probe-test-browser.sh
index 0148594b4953..f0cfb168159a 100755
--- a/yarn-project/kv-store/scripts/probe-test-browser.sh
+++ b/yarn-project/kv-store/scripts/probe-test-browser.sh
@@ -1,12 +1,29 @@
 #!/usr/bin/env bash
-# Temporary diagnostic wrapper for `yarn test:browser` used to capture
-# CI container state around the intermittent hang documented in PR #22693.
-# v2: full log dump, hybrid 1s/0.1s cadence, kernel stacks, socket state.
-# Revert this and the package.local.json change once root-caused.
+# Deep diagnostic wrapper around `yarn test:browser` for investigating
+# browser-test hangs and silent failures.
+#
+# Wraps a single vitest run with a hybrid 1s-steady / 0.1s-burst sampler
+# that captures /proc state (wchan, stack, syscall, fd), cgroup CPU/memory
+# counters, /proc/net socket state counts, and a CPU-availability
+# microbench. When vitest goes silent for >3s the sampler enters dense
+# burst mode and emits a stacks_snapshot every 1s of burst. Outer 90s
+# timeout converts a hang into a clean exit with the full diagnostic dump
+# emitted to stderr by the EXIT trap.
+#
+# Single-tenant invocation:
+#   bash yarn-project/kv-store/scripts/probe-test-browser.sh
+#
+# Parallel invocation (avoids tmp-path collisions across slots):
+#   DIAG_DIR=/tmp/diag.$$ bash yarn-project/kv-store/scripts/probe-test-browser.sh
+#
+# Was used to root-cause the kv-store CDP teardown deadlock that
+# scripts/run-browser-tests.sh now works around; kept for future regression
+# debugging. scripts/repro-browser-hang.sh runs this in a parallel grind
+# under docker_isolate constraints.
 
 cd "$(dirname "$0")/.."
 
-DIAG_DIR=/tmp/diag
+DIAG_DIR=${DIAG_DIR:-/tmp/diag}
 PROBE_LOG=$DIAG_DIR/probe.log
 STACKS_LOG=$DIAG_DIR/stacks.log
 VITEST_LOG=$DIAG_DIR/vitest.log
diff --git a/yarn-project/kv-store/scripts/repro-browser-hang.sh b/yarn-project/kv-store/scripts/repro-browser-hang.sh
new file mode 100755
index 000000000000..84f3ff75f871
--- /dev/null
+++ b/yarn-project/kv-store/scripts/repro-browser-hang.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Reproduces the kv-store browser-test hang under CI3 ISOLATE constraints.
+#
+# Failure mode: CDP CLOSE_WAIT deadlock between vitest and chromium's network
+# service at test-file transitions. Vitest closes 6 of 10 CDP TCP connections
+# on file-boundary teardown; chromium's network-service event loop, starved of
+# CPU under --cpus=2, never drains the closed sockets. Vitest's teardown hangs
+# waiting for the close-handshake. Both processes end up with zero on-CPU
+# threads. Outer `timeout -v 90s` in probe-test-browser.sh fires SIGTERM after
+# 90s of silence.
+#
+# Crucially, this does NOT reproduce on unconstrained hardware — chromium has
+# enough cores to drain sockets immediately. Must run inside docker_isolate
+# (--cpus=2 --memory=8g --tmpfs /tmp:exec,size=1g) to surface the hang.
+#
+# Usage:
+#   bash yarn-project/kv-store/scripts/repro-browser-hang.sh [JOBS]
+#
+#   JOBS: parallel container count (default 8). Higher = faster repro,
+#   higher RAM/CPU ceiling on host. 8 reliably catches a hang in ~3min on
+#   a multi-core box.
+#
+# On hang: parallel halts (rc=124) and the failing job's full probe diagnostic
+# (vitest tail + probe.log + stacks.log) is dumped to this script's stderr by
+# dump_fail. Capture stderr to a file if you want to keep it.
+#
+# Example: capture both streams for later analysis
+#   bash yarn-project/kv-store/scripts/repro-browser-hang.sh 8 > /tmp/repro.log 2>&1
+
+set -uo pipefail
+
+cd "$(git rev-parse --show-toplevel)"
+
+JOBS=${1:-8}
+
+echo "=== repro-browser-hang start: $(date -Is), jobs=$JOBS ==="
+while true; do
+  echo './ci3/dump_fail "CPUS=2 MEM=8g TMPFS_SIZE=1g ./ci3/docker_isolate \"cd yarn-project/kv-store && bash scripts/probe-test-browser.sh\"" >/dev/null'
+done | parallel -j"$JOBS" --halt now,fail=1
+rc=$?
+echo "=== repro-browser-hang end (parallel rc=$rc): $(date -Is) ==="
+exit $rc
diff --git a/yarn-project/kv-store/scripts/run-browser-tests.sh b/yarn-project/kv-store/scripts/run-browser-tests.sh
new file mode 100755
index 000000000000..384becae3dfd
--- /dev/null
+++ b/yarn-project/kv-store/scripts/run-browser-tests.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Runs vitest browser tests one test file per vitest invocation.
+#
+# Why not a single `vitest run` over all files: vitest+chromium have a CDP
+# teardown deadlock at test-file transitions under CPU-constrained
+# environments (CI3 ISOLATE: --cpus=2). Vitest closes a cohort of CDP TCP
+# connections when switching files; chromium's network service can't drain
+# them fast enough under contention; vitest's teardown blocks indefinitely
+# on the close-handshake. See scripts/repro-browser-hang.sh for the repro.
+#
+# By running each file in a separate vitest process, the close-handshake
+# only happens at process exit (where the OS reaps everything), avoiding
+# the cross-file teardown path entirely. Cost is ~5-10s of vite startup
+# per file, which is acceptable for ~10 files.
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+# Mirror vitest.config.ts include patterns. Bench tests are excluded —
+# benchmarks should be run via `yarn bench:browser`, which is already a
+# one-shot invocation.
+files=$(find src/indexeddb src/sqlite-opfs -name '*.test.ts' 2>/dev/null | sort)
+
+if [ -z "$files" ]; then
+  echo "No test files found in src/indexeddb or src/sqlite-opfs"
+  exit 0
+fi
+
+count=$(echo "$files" | wc -l)
+echo "Running $count browser test files (one vitest process per file)"
+
+i=0
+for f in $files; do
+  i=$((i + 1))
+  echo "==> [$i/$count] $f"
+  yarn vitest run --config ./vitest.config.ts "$f"
+done

From 4056a47b3debab82d990880607046196747af230 Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 10:58:34 +0000
Subject: [PATCH 06/11] chore(kv-store): wide-net flake entry routes
 browser-test failures to martin
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Catch-all entry on yarn-project/kv-store with error_regex "vitest" sends
any browser-shaped failure (anything emitting "vitest" in its output) to
martin, so colleagues aren't blocked by residual flakiness while the new
run-browser-tests.sh approach soaks in. Existing narrow entries still
apply — owners are unioned across matching entries — so grego still gets
pinged for his historical patterns.

Intended to be removed after a few weeks of stable CI.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .test_patterns.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.test_patterns.yml b/.test_patterns.yml
index 79cdcb8f3a9a..0d26d00c3984 100644
--- a/.test_patterns.yml
+++ b/.test_patterns.yml
@@ -146,6 +146,16 @@ tests:
       - *palla
 
   # yarn-project tests
+  # Wide net for kv-store browser-test flakes during the bake-in period for
+  # the run-browser-tests.sh fix (CDP teardown deadlock workaround). Routes
+  # any vitest-shaped failure to martin so colleagues aren't blocked by
+  # residual flakiness while we soak the new approach. Narrower entries
+  # below still apply — owners are unioned across matching entries. Remove
+  # this catch-all once the new approach has settled in CI for a few weeks.
+  - regex: "yarn-project/kv-store"
+    error_regex: "vitest"
+    owners:
+      - *martin
   - regex: "yarn-project/kv-store"
     error_regex: "Could not import your test module"
     owners:

From 3b79d9e60d3d439095d8ed544e01ee073211adae Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 11:01:03 +0000
Subject: [PATCH 07/11] chore(kv-store): trim catch-all flake-entry comment

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .test_patterns.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.test_patterns.yml b/.test_patterns.yml
index 0d26d00c3984..afe61faee696 100644
--- a/.test_patterns.yml
+++ b/.test_patterns.yml
@@ -146,12 +146,7 @@ tests:
       - *palla
 
   # yarn-project tests
-  # Wide net for kv-store browser-test flakes during the bake-in period for
-  # the run-browser-tests.sh fix (CDP teardown deadlock workaround). Routes
-  # any vitest-shaped failure to martin so colleagues aren't blocked by
-  # residual flakiness while we soak the new approach. Narrower entries
-  # below still apply — owners are unioned across matching entries. Remove
-  # this catch-all once the new approach has settled in CI for a few weeks.
+  # Attempt to catch all kv-store browser test failures (consider them quarantined for now)
   - regex: "yarn-project/kv-store"
     error_regex: "vitest"
     owners:

From bfb5ca293335c50db7b3f49acd9592ab1ef92607 Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 11:51:34 +0000
Subject: [PATCH 08/11] chore(kv-store): re-run yarn prepare after
 merge-train/fairies merge

The merge brought in next's package.json with "test": "yarn test:node"
(the historical disable workaround), overwriting the chained test set in
the fix commit. package.local.json still has the correct override; this
regenerates package.json from it.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 yarn-project/kv-store/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json
index efeb438cc9b1..b9efae78907a 100644
--- a/yarn-project/kv-store/package.json
+++ b/yarn-project/kv-store/package.json
@@ -18,7 +18,7 @@
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "test:browser": "bash scripts/run-browser-tests.sh",
     "bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
-    "test": "yarn test:node",
+    "test": "yarn test:node && yarn test:browser",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
   "inherits": [

From 035a575ed52132f3ade32cd1cbeebf3890317359 Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 12:09:58 +0000
Subject: [PATCH 09/11] fix(kv-store): mirror VITE_SQLITE_OPFS gate in
 browser-test wrapper
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

vitest.config.ts only includes src/sqlite-opfs/**/*.test.ts when
VITE_SQLITE_OPFS=1 (gated since #22693 due to a separate hang). The
wrapper script enumerated both directories unconditionally, so vitest
received a positional path that the include filter excluded, returned
"no test files found", and exited 1 — killing the loop via set -e.

Mirror the gate at file-enumeration time so the script only feeds
vitest paths that the config will accept.
---
 yarn-project/kv-store/scripts/run-browser-tests.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/yarn-project/kv-store/scripts/run-browser-tests.sh b/yarn-project/kv-store/scripts/run-browser-tests.sh
index 384becae3dfd..05eb1084cc45 100755
--- a/yarn-project/kv-store/scripts/run-browser-tests.sh
+++ b/yarn-project/kv-store/scripts/run-browser-tests.sh
@@ -16,10 +16,15 @@ set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
-# Mirror vitest.config.ts include patterns. Bench tests are excluded —
-# benchmarks should be run via `yarn bench:browser`, which is already a
-# one-shot invocation.
-files=$(find src/indexeddb src/sqlite-opfs -name '*.test.ts' 2>/dev/null | sort)
+# Mirror vitest.config.ts include patterns. sqlite-opfs is gated behind
+# VITE_SQLITE_OPFS=1 in the config (the backend has its own hang under the
+# 2-CPU container; see PR #22693), so we mirror that gate here — otherwise
+# vitest receives a positional path that `include` filters out, finds zero
+# tests, and exits 1. Bench tests are excluded — they run via
+# `yarn bench:browser`, which is already a one-shot invocation.
+dirs=(src/indexeddb)
+[ "${VITE_SQLITE_OPFS:-}" = "1" ] && dirs+=(src/sqlite-opfs)
+files=$(find "${dirs[@]}" -name '*.test.ts' 2>/dev/null | sort)
 
 if [ -z "$files" ]; then
   echo "No test files found in src/indexeddb or src/sqlite-opfs"

From dcd618f00ba4e8dd0dcc693afc5521803200f12f Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 12:33:37 +0000
Subject: [PATCH 10/11] feat(kv-store): re-enable sqlite-opfs browser tests
 under per-file isolation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The VITE_SQLITE_OPFS gate was added in #22693 as a workaround for the
2-CPU CI hang. That hang was a vitest+chromium CDP teardown deadlock
at test-file transitions, now fixed by per-file vitest invocation
(see run-browser-tests.sh). The gating was always meant to be removed
once the hang was root-caused.

Drop the env-flag gate in vitest.config.ts and stop mirroring it in
the wrapper script — vitest now discovers sqlite-opfs tests by default
and the wrapper enumerates them in the same loop as the indexeddb files.

The existing wide-net catch-all in .test_patterns.yml continues to
quarantine any residual flakiness to martin so colleagues aren't blocked.
---
 yarn-project/kv-store/scripts/run-browser-tests.sh | 13 ++++---------
 yarn-project/kv-store/vitest.config.ts             |  7 +------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/yarn-project/kv-store/scripts/run-browser-tests.sh b/yarn-project/kv-store/scripts/run-browser-tests.sh
index 05eb1084cc45..384becae3dfd 100755
--- a/yarn-project/kv-store/scripts/run-browser-tests.sh
+++ b/yarn-project/kv-store/scripts/run-browser-tests.sh
@@ -16,15 +16,10 @@ set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
-# Mirror vitest.config.ts include patterns. sqlite-opfs is gated behind
-# VITE_SQLITE_OPFS=1 in the config (the backend has its own hang under the
-# 2-CPU container; see PR #22693), so we mirror that gate here — otherwise
-# vitest receives a positional path that `include` filters out, finds zero
-# tests, and exits 1. Bench tests are excluded — they run via
-# `yarn bench:browser`, which is already a one-shot invocation.
-dirs=(src/indexeddb)
-[ "${VITE_SQLITE_OPFS:-}" = "1" ] && dirs+=(src/sqlite-opfs)
-files=$(find "${dirs[@]}" -name '*.test.ts' 2>/dev/null | sort)
+# Mirror vitest.config.ts include patterns. Bench tests are excluded —
+# benchmarks should be run via `yarn bench:browser`, which is already a
+# one-shot invocation.
+files=$(find src/indexeddb src/sqlite-opfs -name '*.test.ts' 2>/dev/null | sort)
 
 if [ -z "$files" ]; then
   echo "No test files found in src/indexeddb or src/sqlite-opfs"
diff --git a/yarn-project/kv-store/vitest.config.ts b/yarn-project/kv-store/vitest.config.ts
index cee6d7f49cd1..7e73fbcc72a2 100644
--- a/yarn-project/kv-store/vitest.config.ts
+++ b/yarn-project/kv-store/vitest.config.ts
@@ -52,12 +52,7 @@ export default defineConfig({
     reporters: ['verbose'],
     include: [
       './src/indexeddb/**/*.test.ts',
-      // sqlite-opfs browser tests hang intermittently in the 2-CPU CI container
-      // (see https://github.com/AztecProtocol/aztec-packages/pull/22693). The
-      // backend is still experimental — gate these behind VITE_SQLITE_OPFS=1
-      // so they run in dev but don't block the merge-train until the hang is
-      // root-caused.
-      ...(process.env.VITE_SQLITE_OPFS === '1' ? ['./src/sqlite-opfs/**/*.test.ts'] : []),
+      './src/sqlite-opfs/**/*.test.ts',
       // Benchmarks self-skip unless VITE_BENCH=1; include so they're discoverable.
       './src/bench/indexeddb/**/*.test.ts',
       './src/bench/sqlite-opfs/**/*.test.ts',

From 5d28e3769ee2cf8f4b8ed4e3cd698c04f1a33ad7 Mon Sep 17 00:00:00 2001
From: mverzilli <martin@aztec-labs.com>
Date: Tue, 28 Apr 2026 13:19:56 +0000
Subject: [PATCH 11/11] feat(kv-store): emit one test_cmds line per file for CI
 per-file fan-out
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CI's bootstrap test_cmds machinery distributes each emitted command to a
separate parallel slot. kv-store was previously emitted as a single line
(`cd yarn-project/kv-store && yarn test`), which collapsed all 28 of its
test files into one slot — leaving the rest of the executor idle while
the package serialised through every file.

Add kv-store/bootstrap.sh with a test_cmds function that emits one line
per test file, mirroring the aztec/bootstrap.sh pattern. New
kv-store/scripts/run_test.sh dispatches a single file to vitest (browser
tests under indexeddb, sqlite-opfs) or mocha (everything else). Browser
files keep ISOLATE=1; node files run unisolated like other packages.

The local-dev `yarn test:browser` path (run-browser-tests.sh) now also
goes through run_test.sh, so per-file invocation has a single source of
truth for both CI and local reproduction.
---
 yarn-project/bootstrap.sh                     |  6 ++--
 yarn-project/kv-store/bootstrap.sh            | 28 +++++++++++++++++++
 .../kv-store/scripts/run-browser-tests.sh     | 18 +++++-------
 yarn-project/kv-store/scripts/run_test.sh     | 20 +++++++++++++
 4 files changed, 58 insertions(+), 14 deletions(-)
 create mode 100755 yarn-project/kv-store/bootstrap.sh
 create mode 100755 yarn-project/kv-store/scripts/run_test.sh

diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh
index 75c3aacbce73..28cc9d41619f 100755
--- a/yarn-project/bootstrap.sh
+++ b/yarn-project/bootstrap.sh
@@ -170,7 +170,7 @@ function test_cmds {
 
   # Exclusions:
   # end-to-end: e2e tests handled separately with end-to-end/bootstrap.sh.
-  # kv-store: Uses mocha so will need different treatment.
+  # kv-store: per-file fan-out handled by kv-store/bootstrap.sh test_cmds.
   for test in !(end-to-end|kv-store|aztec)/src/**/*.test.ts; do
     # Skip benchmarks here.
     [[ "$test" =~ \.bench\.test\.ts$ ]] && continue
@@ -218,8 +218,8 @@ function test_cmds {
     echo "${prefix}${cmd_env} yarn-project/scripts/run_test.sh $test"
   done
 
-  # Uses mocha for browser tests, so we have to treat it differently.
-  echo "$hash:ISOLATE=1 cd yarn-project/kv-store && yarn test"
+  # kv-store: per-file fan-out (mocha for node tests, vitest for browser tests).
+  kv-store/bootstrap.sh test_cmds
 
   # Aztec CLI tests
   aztec/bootstrap.sh test_cmds
diff --git a/yarn-project/kv-store/bootstrap.sh b/yarn-project/kv-store/bootstrap.sh
new file mode 100755
index 000000000000..0f4d344a8812
--- /dev/null
+++ b/yarn-project/kv-store/bootstrap.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
+
+hash=$(../bootstrap.sh hash)
+
+function test_cmds {
+  # Node tests (mocha): files outside the browser-test and bench dirs.
+  # Mirrors .mocharc.json's spec.
+  for test in src/**/!(indexeddb|sqlite-opfs|bench)/*.test.ts; do
+    echo "$hash yarn-project/kv-store/scripts/run_test.sh $test"
+  done
+
+  # Browser tests (vitest + chromium). Each file runs in its own ISOLATE
+  # container — running multiple files in a single vitest invocation
+  # triggers a CDP teardown deadlock on the 2-CPU CI executor. See
+  # scripts/run-browser-tests.sh for the root-cause analysis.
+  for test in src/indexeddb/*.test.ts src/sqlite-opfs/*.test.ts; do
+    echo "$hash:ISOLATE=1 yarn-project/kv-store/scripts/run_test.sh $test"
+  done
+}
+
+case "$cmd" in
+  "")
+    ;;
+  *)
+    default_cmd_handler "$@"
+    ;;
+esac
diff --git a/yarn-project/kv-store/scripts/run-browser-tests.sh b/yarn-project/kv-store/scripts/run-browser-tests.sh
index 384becae3dfd..b13f9f662c28 100755
--- a/yarn-project/kv-store/scripts/run-browser-tests.sh
+++ b/yarn-project/kv-store/scripts/run-browser-tests.sh
@@ -1,24 +1,20 @@
 #!/usr/bin/env bash
-# Runs vitest browser tests one test file per vitest invocation.
+# Local-dev entrypoint for `yarn test:browser`: runs every browser test file
+# in its own vitest process, sequentially. Delegates per-file dispatch to
+# scripts/run_test.sh (which CI also uses for per-file fan-out).
 #
 # Why not a single `vitest run` over all files: vitest+chromium have a CDP
 # teardown deadlock at test-file transitions under CPU-constrained
 # environments (CI3 ISOLATE: --cpus=2). Vitest closes a cohort of CDP TCP
 # connections when switching files; chromium's network service can't drain
 # them fast enough under contention; vitest's teardown blocks indefinitely
-# on the close-handshake. See scripts/repro-browser-hang.sh for the repro.
-#
-# By running each file in a separate vitest process, the close-handshake
-# only happens at process exit (where the OS reaps everything), avoiding
-# the cross-file teardown path entirely. Cost is ~5-10s of vite startup
-# per file, which is acceptable for ~10 files.
+# on the close-handshake. By running each file in a separate vitest process
+# the close-handshake only happens at process exit, avoiding the cross-file
+# teardown path entirely. See scripts/repro-browser-hang.sh for the repro.
 set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
-# Mirror vitest.config.ts include patterns. Bench tests are excluded —
-# benchmarks should be run via `yarn bench:browser`, which is already a
-# one-shot invocation.
 files=$(find src/indexeddb src/sqlite-opfs -name '*.test.ts' 2>/dev/null | sort)
 
 if [ -z "$files" ]; then
@@ -33,5 +29,5 @@ i=0
 for f in $files; do
   i=$((i + 1))
   echo "==> [$i/$count] $f"
-  yarn vitest run --config ./vitest.config.ts "$f"
+  bash scripts/run_test.sh "$f"
 done
diff --git a/yarn-project/kv-store/scripts/run_test.sh b/yarn-project/kv-store/scripts/run_test.sh
new file mode 100755
index 000000000000..999aa9dd3271
--- /dev/null
+++ b/yarn-project/kv-store/scripts/run_test.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Runs a single kv-store test file. Dispatches to vitest+chromium for
+# browser tests (under src/indexeddb or src/sqlite-opfs) and to mocha for
+# everything else. Emitted by yarn-project/kv-store/bootstrap.sh test_cmds
+# for CI per-file fan-out and runnable directly for local reproduction:
+#
+#   yarn-project/kv-store/scripts/run_test.sh src/lmdb-v2/store.test.ts
+source $(git rev-parse --show-toplevel)/ci3/source
+
+test=${1:?"Usage: $0 <test-file relative to kv-store/>"}
+cd ..
+
+case "$test" in
+  src/indexeddb/*|src/sqlite-opfs/*)
+    exec yarn vitest run --config ./vitest.config.ts "$test"
+    ;;
+  *)
+    NODE_NO_WARNINGS=1 exec yarn mocha --config ./.mocharc.json "$test"
+    ;;
+esac