diff --git a/ci3/docker_isolate b/ci3/docker_isolate index 868ea59c8e9b..7ca1ab5f4fa2 100755 --- a/ci3/docker_isolate +++ b/ci3/docker_isolate @@ -10,6 +10,9 @@ export TMPFS_SIZE=${TMPFS_SIZE:-1g} export USE_HOME_TMP=${USE_HOME_TMP:-0} function cleanup { + if [ -f "${run_err:-}" ]; then + rm -f "$run_err" + fi if [ -n "${cid:-}" ]; then docker rm -f "$cid" &>/dev/null fi @@ -24,17 +27,6 @@ if [ -n "${NAME:-}" ]; then name_arg="--name $name" # Kill any existing container with the same name. docker rm -f "$name" &>/dev/null || true - # Under host load, dockerd can leave a container "Dead" or mid-removal for several - # seconds after `rm -f` returns, so wait for it to actually disappear before reusing - # the name below. Otherwise `docker run --name` races into "Conflict, name already in use". - for _ in $(seq 1 30); do - docker container inspect "$name" &>/dev/null || break - sleep 1 - done - if docker container inspect "$name" &>/dev/null; then - echo "docker_isolate: dockerd failed to remove stale container $name within 30s" >&2 - exit 1 - fi fi # For pinning to given CPUs. @@ -66,33 +58,57 @@ fi # Launch the container in the background. # Don't launch in the foreground or you can't process SIGINT/SIGTERM. # Don't use & as we want to block, and be sure it starts before processing any signals. -set -x -cid=$(docker run -d \ - ${name_arg:-} \ - ${network_arg:-} \ - ${cpuset_arg:-} \ - --cpus=$CPUS \ - --memory=$MEM \ - --user $(id -u):$(id -g) \ - -v$HOME:$HOME \ - $tmp_mount \ - --workdir $PWD \ - -e HOME \ - -e VERBOSE \ - -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \ - -e FORCE_COLOR=true \ - -e CPUS \ - -e MEM \ - -e TMPFS_SIZE \ - -e USE_HOME_TMP \ - -e AVM \ - -e ELU_MONITOR_FILE \ - -e TEST_TIMING_FILE \ - -e RUN_ID \ - "${arg_env_vars[@]}" \ - aztecprotocol/build:3.0 \ - /bin/bash -c "$cmd") -set +x +# Under host load, dockerd can release a removed container's name several seconds after +# `rm -f` returns (teardown/name-registry race), so `docker run --name` can hit +# "Conflict, name already in use". Retry with backoff (~30s total), force-removing +# the name's holder between attempts. +run_err=$(mktemp) +max_attempts=6 +for attempt in $(seq 1 $max_attempts); do + set -x +e + cid=$(docker run -d \ + ${name_arg:-} \ + ${network_arg:-} \ + ${cpuset_arg:-} \ + --cpus=$CPUS \ + --memory=$MEM \ + --user $(id -u):$(id -g) \ + -v$HOME:$HOME \ + $tmp_mount \ + --workdir $PWD \ + -e HOME \ + -e VERBOSE \ + -e GIT_CONFIG_GLOBAL=$root/build-images/src/home/.gitconfig \ + -e FORCE_COLOR=true \ + -e CPUS \ + -e MEM \ + -e TMPFS_SIZE \ + -e USE_HOME_TMP \ + -e AVM \ + -e ELU_MONITOR_FILE \ + -e TEST_TIMING_FILE \ + -e RUN_ID \ + "${arg_env_vars[@]}" \ + aztecprotocol/build:3.0 \ + /bin/bash -c "$cmd" 2>"$run_err") + run_code=$? + set +x -e + # Surface docker's stderr (warnings on success, the error on failure). + if [ -s "$run_err" ]; then + cat "$run_err" >&2 + fi + if [ "$run_code" -eq 0 ]; then + break + fi + if [ "$attempt" -lt "$max_attempts" ] && [ -n "${name:-}" ] \ + && grep -qi "already in use by container" "$run_err"; then + echo "docker_isolate: name conflict for $name (attempt $attempt/$max_attempts), removing and retrying" >&2 + docker rm -f "$name" &>/dev/null || true + sleep $((1 << (attempt - 1))) + continue + fi + exit "$run_code" +done # At this point we know it's started, and can handle SIGTERM to kill the container. # Output logs, but use & so we can still handle signals. diff --git a/ci3/run_test_cmd b/ci3/run_test_cmd index a84b7dc3243f..34f8f9425471 100755 --- a/ci3/run_test_cmd +++ b/ci3/run_test_cmd @@ -422,6 +422,11 @@ else echo -e "${yellow}RETRYING${reset}${log_info}: $test_cmd" + # Give the retry fresh container/compose-project names. The failed attempt's containers can + # still be mid-teardown in dockerd, and reusing the same name races into "Conflict, name + # already in use" (docker_isolate and run_compose_test append NAME_POSTFIX to their names). + export NAME_POSTFIX="${NAME_POSTFIX:-}_retry_$(uuid)" + run_test if [ $code -eq 0 ]; then diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index 1e0b5149d626..8ea2bb11b9c7 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -53,13 +53,6 @@ function test_cmds { local tests=( # List all standalone and nested tests, except for the ones listed above. src/automine/*.test.ts - src/automine/contracts/*.test.ts - src/automine/contracts/deploy/*.test.ts - src/automine/contracts/nested/*.test.ts - src/automine/token/*.test.ts - src/automine/accounts/*.test.ts - src/automine/effects/*.test.ts - src/automine/delivery/*.test.ts src/automine/!(simulation)/**/*.test.ts src/automine/simulation/!(avm_simulator).test.ts src/single-node/!(prover)/**/*.test.ts