Retry failed container removals (JVMHookResourceReaper)

Signed-off-by: Michael Edgar <medgar@redhat.com>

Author: MikeEdgar

core/src/main/java/org/testcontainers/utility/JVMHookResourceReaper.java

@@ -1,9 +1,14 @@
 package org.testcontainers.utility;
 
+import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.model.Container;
 import com.github.dockerjava.api.model.PruneType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -13,19 +18,34 @@
  */
 class JVMHookResourceReaper extends ResourceReaper {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(JVMHookResourceReaper.class);
+
     @Override
     public void init() {
         setHook();
     }
 
+    /**
+     * Perform a cleanup.
+     * @deprecated no longer supported API, use {@link DockerClient} directly
+     */
     @Override
+    @Deprecated
     public synchronized void performCleanup() {
         super.performCleanup();
         synchronized (DEATH_NOTE) {
-            DEATH_NOTE.forEach(filters -> prune(PruneType.CONTAINERS, filters));
-            DEATH_NOTE.forEach(filters -> prune(PruneType.NETWORKS, filters));
-            DEATH_NOTE.forEach(filters -> prune(PruneType.VOLUMES, filters));
-            DEATH_NOTE.forEach(filters -> prune(PruneType.IMAGES, filters));
+            tryPrune(PruneType.CONTAINERS);
+            tryPrune(PruneType.NETWORKS);
+            tryPrune(PruneType.VOLUMES);
+            tryPrune(PruneType.IMAGES);
+        }
+    }
+
+    private void tryPrune(PruneType pruneType) {
+        try {
+            DEATH_NOTE.forEach(filters -> prune(pruneType, filters));
+        } catch (Exception e) {
+            LOGGER.warn("Exception pruning {} resources: {}", pruneType, e.getMessage(), e);
         }
     }
 
@@ -35,28 +55,45 @@ private void prune(PruneType pruneType, List<Map.Entry<String, String>> filters)
             .filter(it -> "label".equals(it.getKey()))
             .map(Map.Entry::getValue)
             .toArray(String[]::new);
-        switch (pruneType) {
+
+        if (pruneType == PruneType.CONTAINERS) {
             // Docker only prunes stopped containers, so we have to do it manually
-            case CONTAINERS:
-                List<Container> containers = dockerClient
-                    .listContainersCmd()
-                    .withFilter("label", Arrays.asList(labels))
-                    .withShowAll(true)
-                    .exec();
-
-                containers
-                    .parallelStream()
-                    .forEach(container -> {
-                        dockerClient
-                            .removeContainerCmd(container.getId())
-                            .withForce(true)
-                            .withRemoveVolumes(true)
-                            .exec();
-                    });
-                break;
-            default:
-                dockerClient.pruneCmd(pruneType).withLabelFilter(labels).exec();
-                break;
+            removeContainers(labels);
+        } else {
+            dockerClient.pruneCmd(pruneType).withLabelFilter(labels).exec();
+        }
+    }
+
+    private void removeContainers(String[] labels) {
+        List<Container> containers = listContainers(labels);
+        int retries = 5;
+
+        while (!containers.isEmpty() && retries-- > 0) {
+            List<Throwable> errors = new ArrayList<>();
+
+            containers.parallelStream().forEach(container -> removeContainer(container, errors));
+
+            if (errors.isEmpty()) {
+                containers = Collections.emptyList();
+            } else if (retries < 1) {
+                RuntimeException removeError = new RuntimeException("Error removing one or more containers");
+                errors.forEach(removeError::addSuppressed);
+                throw removeError;
+            } else {
+                containers = listContainers(labels);
+            }
+        }
+    }
+
+    private List<Container> listContainers(String[] labels) {
+        return dockerClient.listContainersCmd().withFilter("label", Arrays.asList(labels)).withShowAll(true).exec();
+    }
+
+    private void removeContainer(Container container, List<Throwable> errors) {
+        try {
+            dockerClient.removeContainerCmd(container.getId()).withForce(true).withRemoveVolumes(true).exec();
+        } catch (Exception e) {
+            errors.add(e);
         }
     }
 }