fix(ci): retry docker run on name conflicts and give test retries fresh container names

ci3/docker_isolate

@@ -10,6 +10,9 @@ export TMPFS_SIZE=${TMPFS_SIZE:-1g}
 export USE_HOME_TMP=${USE_HOME_TMP:-0}
 
 function cleanup {
+  if [ -f "${run_err:-}" ]; then
+    rm -f "$run_err"
+  fi
   if [ -n "${cid:-}" ]; then
     docker rm -f "$cid" &>/dev/null
   fi
@@ -24,17 +27,6 @@ if [ -n "${NAME:-}" ]; then
   name_arg="--name $name"
   # Kill any existing container with the same name.
   docker rm -f "$name" &>/dev/null || true
-  # Under host load, dockerd can leave a container "Dead" or mid-removal for several
-  # seconds after `rm -f` returns, so wait for it to actually disappear before reusing
-  # the name below. Otherwise `docker run --name` races into "Conflict, name already in use".
-  for _ in $(seq 1 30); do
-    docker container inspect "$name" &>/dev/null || break
-    sleep 1
-  done
-  if docker container inspect "$name" &>/dev/null; then
-    echo "docker_isolate: dockerd failed to remove stale container $name within 30s" >&2
-    exit 1
-  fi
 fi
 
 # For pinning to given CPUs.
@@ -66,33 +58,57 @@ fi
 # Launch the container in the background.
 # Don't launch in the foreground or you can't process SIGINT/SIGTERM.
 # Don't use & as we want to block, and be sure it starts before processing any signals.
-set -x
-cid=$(docker run -d \
-  ${name_arg:-} \
-  ${network_arg:-} \
-  ${cpuset_arg:-} \
-  --cpus=$CPUS \
-  --memory=$MEM \
-  --user $(id -u):$(id -g) \
-  -v$HOME:$HOME \
-  $tmp_mount \
-  --workdir $PWD \
-  -e HOME \
-  -e VERBOSE \
-  -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \
-  -e FORCE_COLOR=true \
-  -e CPUS \
-  -e MEM \
-  -e TMPFS_SIZE \
-  -e USE_HOME_TMP \
-  -e AVM \
-  -e ELU_MONITOR_FILE \
-  -e TEST_TIMING_FILE \
-  -e RUN_ID \
-  "${arg_env_vars[@]}" \
-  aztecprotocol/build:3.0 \
-  /bin/bash -c "$cmd")
-set +x
+# Under host load, dockerd can release a removed container's name several seconds after
+# `rm -f` returns (teardown/name-registry race), so `docker run --name` can hit
+# "Conflict, name already in use". Retry with backoff (~30s total), force-removing
+# the name's holder between attempts.
+run_err=$(mktemp)
+max_attempts=6
+for attempt in $(seq 1 $max_attempts); do
+  set -x +e
+  cid=$(docker run -d \
+    ${name_arg:-} \
+    ${network_arg:-} \
+    ${cpuset_arg:-} \
+    --cpus=$CPUS \
+    --memory=$MEM \
+    --user $(id -u):$(id -g) \
+    -v$HOME:$HOME \
+    $tmp_mount \
+    --workdir $PWD \
+    -e HOME \
+    -e VERBOSE \
+    -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \
+    -e FORCE_COLOR=true \
+    -e CPUS \
+    -e MEM \
+    -e TMPFS_SIZE \
+    -e USE_HOME_TMP \
+    -e AVM \
+    -e ELU_MONITOR_FILE \
+    -e TEST_TIMING_FILE \
+    -e RUN_ID \
+    "${arg_env_vars[@]}" \
+    aztecprotocol/build:3.0 \
+    /bin/bash -c "$cmd" 2>"$run_err")
+  run_code=$?
+  set +x -e
+  # Surface docker's stderr (warnings on success, the error on failure).
+  if [ -s "$run_err" ]; then
+    cat "$run_err" >&2
+  fi
+  if [ "$run_code" -eq 0 ]; then
+    break
+  fi
+  if [ "$attempt" -lt "$max_attempts" ] && [ -n "${name:-}" ] \
+      && grep -qi "already in use by container" "$run_err"; then
+    echo "docker_isolate: name conflict for $name (attempt $attempt/$max_attempts), removing and retrying" >&2
+    docker rm -f "$name" &>/dev/null || true
+    sleep $((1 << (attempt - 1)))
+    continue
+  fi
+  exit "$run_code"
+done
 # At this point we know it's started, and can handle SIGTERM to kill the container.
 
 # Output logs, but use & so we can still handle signals.

ci3/run_test_cmd

@@ -422,6 +422,11 @@ else
 
   echo -e "${yellow}RETRYING${reset}${log_info}: $test_cmd"
 
+  # Give the retry fresh container/compose-project names. The failed attempt's containers can
+  # still be mid-teardown in dockerd, and reusing the same name races into "Conflict, name
+  # already in use" (docker_isolate and run_compose_test append NAME_POSTFIX to their names).
+  export NAME_POSTFIX="${NAME_POSTFIX:-}_retry_$(uuid)"
+
   run_test
 
   if [ $code -eq 0 ]; then

yarn-project/end-to-end/bootstrap.sh

@@ -53,13 +53,6 @@ function test_cmds {
   local tests=(
     # List all standalone and nested tests, except for the ones listed above.
     src/automine/*.test.ts
-    src/automine/contracts/*.test.ts
-    src/automine/contracts/deploy/*.test.ts
-    src/automine/contracts/nested/*.test.ts
-    src/automine/token/*.test.ts
-    src/automine/accounts/*.test.ts
-    src/automine/effects/*.test.ts
-    src/automine/delivery/*.test.ts
     src/automine/!(simulation)/**/*.test.ts
     src/automine/simulation/!(avm_simulator).test.ts
     src/single-node/!(prover)/**/*.test.ts