From 2ad18d3d92425f3fd16ee6fdff1acdca418e52f0 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 2 Jul 2026 16:00:53 -0400 Subject: [PATCH 1/3] fix(ci): retry docker run on container name conflicts and give test retries fresh names The polling loop from #24469 waits for a stale same-name container to disappear before docker run, but dockerd can still release the name a beat after the container stops being inspectable. Wrap docker run in a bounded retry that force-removes the name's holder on 'already in use' conflicts. Also export a unique NAME_POSTFIX before the flake retry in run_test_cmd so the retry's containers/compose projects never contend with the failed attempt's teardown in the first place. --- ci3/docker_isolate | 81 ++++++++++++++++++++++++++++++---------------- ci3/run_test_cmd | 5 +++ 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/ci3/docker_isolate b/ci3/docker_isolate index 868ea59c8e9b..18909122b82c 100755 --- a/ci3/docker_isolate +++ b/ci3/docker_isolate @@ -10,6 +10,9 @@ export TMPFS_SIZE=${TMPFS_SIZE:-1g} export USE_HOME_TMP=${USE_HOME_TMP:-0} function cleanup { + if [ -f "${run_err:-}" ]; then + rm -f "$run_err" + fi if [ -n "${cid:-}" ]; then docker rm -f "$cid" &>/dev/null fi @@ -66,33 +69,57 @@ fi # Launch the container in the background. # Don't launch in the foreground or you can't process SIGINT/SIGTERM. # Don't use & as we want to block, and be sure it starts before processing any signals. -set -x -cid=$(docker run -d \ - ${name_arg:-} \ - ${network_arg:-} \ - ${cpuset_arg:-} \ - --cpus=$CPUS \ - --memory=$MEM \ - --user $(id -u):$(id -g) \ - -v$HOME:$HOME \ - $tmp_mount \ - --workdir $PWD \ - -e HOME \ - -e VERBOSE \ - -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \ - -e FORCE_COLOR=true \ - -e CPUS \ - -e MEM \ - -e TMPFS_SIZE \ - -e USE_HOME_TMP \ - -e AVM \ - -e ELU_MONITOR_FILE \ - -e TEST_TIMING_FILE \ - -e RUN_ID \ - "${arg_env_vars[@]}" \ - aztecprotocol/build:3.0 \ - /bin/bash -c "$cmd") -set +x +# Even after the inspect wait above, dockerd can release the old container's name a beat after +# the container stops being inspectable (teardown/name-registry race), so `docker run --name` +# can still hit "Conflict, name already in use". Retry a bounded number of times, force-removing +# the name's holder between attempts. +run_err=$(mktemp) +max_attempts=5 +for attempt in $(seq 1 $max_attempts); do + set -x +e + cid=$(docker run -d \ + ${name_arg:-} \ + ${network_arg:-} \ + ${cpuset_arg:-} \ + --cpus=$CPUS \ + --memory=$MEM \ + --user $(id -u):$(id -g) \ + -v$HOME:$HOME \ + $tmp_mount \ + --workdir $PWD \ + -e HOME \ + -e VERBOSE \ + -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \ + -e FORCE_COLOR=true \ + -e CPUS \ + -e MEM \ + -e TMPFS_SIZE \ + -e USE_HOME_TMP \ + -e AVM \ + -e ELU_MONITOR_FILE \ + -e TEST_TIMING_FILE \ + -e RUN_ID \ + "${arg_env_vars[@]}" \ + aztecprotocol/build:3.0 \ + /bin/bash -c "$cmd" 2>"$run_err") + run_code=$? + set +x -e + # Surface docker's stderr (warnings on success, the error on failure). + if [ -s "$run_err" ]; then + cat "$run_err" >&2 + fi + if [ "$run_code" -eq 0 ]; then + break + fi + if [ "$attempt" -lt "$max_attempts" ] && [ -n "${name:-}" ] \ + && grep -qi "already in use by container" "$run_err"; then + echo "docker_isolate: name conflict for $name (attempt $attempt/$max_attempts), removing and retrying" >&2 + docker rm -f "$name" &>/dev/null || true + sleep 1 + continue + fi + exit "$run_code" +done # At this point we know it's started, and can handle SIGTERM to kill the container. # Output logs, but use & so we can still handle signals. diff --git a/ci3/run_test_cmd b/ci3/run_test_cmd index a84b7dc3243f..34f8f9425471 100755 --- a/ci3/run_test_cmd +++ b/ci3/run_test_cmd @@ -422,6 +422,11 @@ else echo -e "${yellow}RETRYING${reset}${log_info}: $test_cmd" + # Give the retry fresh container/compose-project names. The failed attempt's containers can + # still be mid-teardown in dockerd, and reusing the same name races into "Conflict, name + # already in use" (docker_isolate and run_compose_test append NAME_POSTFIX to their names). + export NAME_POSTFIX="${NAME_POSTFIX:-}_retry_$(uuid)" + run_test if [ $code -eq 0 ]; then From b3327064adfc66523a550eb4cb140e3c8bd72ae6 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 2 Jul 2026 16:36:09 -0400 Subject: [PATCH 2/3] fix(ci): drop redundant stale-name polling from docker_isolate The #24469 inspect-poll waited up to 30s for a stale container's name to free before docker run, but the retry loop now covers the same race (including the window after the container stops being inspectable) while actively force-removing the name's holder between attempts. Keep the initial best-effort docker rm -f so the common stale-container case does not burn a retry, and give the retry loop exponential backoff (1+2+4+8+16s, ~30s total) to preserve the old tolerance window. --- ci3/docker_isolate | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/ci3/docker_isolate b/ci3/docker_isolate index 18909122b82c..7ca1ab5f4fa2 100755 --- a/ci3/docker_isolate +++ b/ci3/docker_isolate @@ -27,17 +27,6 @@ if [ -n "${NAME:-}" ]; then name_arg="--name $name" # Kill any existing container with the same name. docker rm -f "$name" &>/dev/null || true - # Under host load, dockerd can leave a container "Dead" or mid-removal for several - # seconds after `rm -f` returns, so wait for it to actually disappear before reusing - # the name below. Otherwise `docker run --name` races into "Conflict, name already in use". - for _ in $(seq 1 30); do - docker container inspect "$name" &>/dev/null || break - sleep 1 - done - if docker container inspect "$name" &>/dev/null; then - echo "docker_isolate: dockerd failed to remove stale container $name within 30s" >&2 - exit 1 - fi fi # For pinning to given CPUs. @@ -69,12 +58,12 @@ fi # Launch the container in the background. # Don't launch in the foreground or you can't process SIGINT/SIGTERM. # Don't use & as we want to block, and be sure it starts before processing any signals. -# Even after the inspect wait above, dockerd can release the old container's name a beat after -# the container stops being inspectable (teardown/name-registry race), so `docker run --name` -# can still hit "Conflict, name already in use". Retry a bounded number of times, force-removing +# Under host load, dockerd can release a removed container's name several seconds after +# `rm -f` returns (teardown/name-registry race), so `docker run --name` can hit +# "Conflict, name already in use". Retry with backoff (~30s total), force-removing # the name's holder between attempts. run_err=$(mktemp) -max_attempts=5 +max_attempts=6 for attempt in $(seq 1 $max_attempts); do set -x +e cid=$(docker run -d \ @@ -115,7 +104,7 @@ for attempt in $(seq 1 $max_attempts); do && grep -qi "already in use by container" "$run_err"; then echo "docker_isolate: name conflict for $name (attempt $attempt/$max_attempts), removing and retrying" >&2 docker rm -f "$name" &>/dev/null || true - sleep 1 + sleep $((1 << (attempt - 1))) continue fi exit "$run_code" From f80d7e47f7b319f4504cf424c66ac0ca69c05149 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Mon, 6 Jul 2026 13:11:40 -0400 Subject: [PATCH 3/3] fix(e2e): dedupe automine test cmds causing mass container-kill flakes (#24547) Stacked on #24477. Fixes the 49-per-run automine FLAKEs reported there and on this PR's first run. **Root cause** (`yarn-project/end-to-end/bootstrap.sh`): the `tests` array listed the automine subdirectories explicitly and also kept the `src/automine/!(simulation)/**/*.test.ts` catch-all (a merge artifact from merging merge-train/fairies-v5 into mv/e2e-onchain-delivery-harness), so every nested automine test was emitted as two byte-identical cmds (268 jobs for 145 tests). Both copies run under the same docker container name. When the two copies overlap in time, the later one's docker_isolate force-removes the earlier one's live container (or wins the name-conflict retry added in #24477), the earlier test fails, and its automatic retry (which gets a fresh `NAME_POSTFIX`) passes: a deterministic FLAKED-with-code-0 for every overlapping pair, which is why the count sits near 49 on every run and why only `src/automine/**` flakes. When the copies do not overlap, the second is skipped by the test cache, hiding the duplication. The parent branch's "docker container name conflicts" were the same duplication seen from the other side. - Remove the seven redundant per-directory globs; the deduplicated unique cmd set is unchanged (verified byte-identical before/after, 145 automine jobs, zero duplicates in both `CI_FULL` modes). --- yarn-project/end-to-end/bootstrap.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index 1e0b5149d626..8ea2bb11b9c7 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -53,13 +53,6 @@ function test_cmds { local tests=( # List all standalone and nested tests, except for the ones listed above. src/automine/*.test.ts - src/automine/contracts/*.test.ts - src/automine/contracts/deploy/*.test.ts - src/automine/contracts/nested/*.test.ts - src/automine/token/*.test.ts - src/automine/accounts/*.test.ts - src/automine/effects/*.test.ts - src/automine/delivery/*.test.ts src/automine/!(simulation)/**/*.test.ts src/automine/simulation/!(avm_simulator).test.ts src/single-node/!(prover)/**/*.test.ts