Pooled Test Containers (#329)

the recent test flakiness is happening because we are spinning up a
unique PG container for each test. This PR changes that to a pool of 4
containers, but we create a new DB for each test and drop the DB on test
completion.

Updated the junit default to run tests concurrently unless otherwise
annotated and to run 4 tests in parallel maximum.

DRYed out test reporing in the build files + added logic to pring the
failing tests at the end of the test run for easier diagnosis

Author: devhawk

build.gradle.kts

@@ -157,6 +157,44 @@ subprojects {
     // use the environment's JDK instead of the toolchain's JDK for tests
     tasks.withType<Test> { javaLauncher.set(null as JavaLauncher?) }
 
+    tasks.withType<Test> {
+      useJUnitPlatform()
+      testLogging {
+        events("passed", "skipped", "failed")
+        showStandardStreams = true
+      }
+      addTestListener(
+        object : TestListener {
+          private val failedTests = mutableListOf<String>()
+
+          override fun beforeSuite(suite: TestDescriptor) {}
+
+          override fun beforeTest(testDescriptor: TestDescriptor) {}
+
+          override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {
+            if (result.resultType == TestResult.ResultType.FAILURE) {
+              failedTests.add("${testDescriptor.className}.${testDescriptor.name}")
+            }
+          }
+
+          override fun afterSuite(suite: TestDescriptor, result: TestResult) {
+            if (suite.parent == null) {
+              println("\nTest Results:")
+              println("  Tests run: ${result.testCount}")
+              println("  Passed: ${result.successfulTestCount}")
+              println("  Failed: ${result.failedTestCount}")
+              println("  Skipped: ${result.skippedTestCount}")
+
+              if (failedTests.isNotEmpty()) {
+                println("\nFailed Tests:")
+                failedTests.forEach { println("  - $it") }
+              }
+            }
+          }
+        }
+      )
+    }
+
     tasks.named<Jar>("jar") {
       manifest {
         attributes["Implementation-Version"] = project.version

transact-cli/build.gradle.kts

@@ -18,34 +18,6 @@ dependencies {
   testImplementation("org.testcontainers:testcontainers-postgresql:2.0.3")
 }
 
-tasks.test {
-  useJUnitPlatform()
-  testLogging {
-    events("passed", "skipped", "failed")
-    showStandardStreams = true
-  }
-
-  addTestListener(
-    object : TestListener {
-      override fun beforeSuite(suite: TestDescriptor) {}
-
-      override fun beforeTest(testDescriptor: TestDescriptor) {}
-
-      override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {}
-
-      override fun afterSuite(suite: TestDescriptor, result: TestResult) {
-        if (suite.parent == null) {
-          println("\nTest Results:")
-          println("  Tests run: ${result.testCount}")
-          println("  Passed: ${result.successfulTestCount}")
-          println("  Failed: ${result.failedTestCount}")
-          println("  Skipped: ${result.skippedTestCount}")
-        }
-      }
-    }
-  )
-}
-
 tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar") {
   archiveBaseName.set("dbos")
   archiveVersion.set("")

transact-cli/src/test/java/dev/dbos/transact/cli/MigrateCommandTest.java

@@ -20,7 +20,6 @@
 import picocli.CommandLine;
 
 @org.junit.jupiter.api.Timeout(value = 2, unit = java.util.concurrent.TimeUnit.MINUTES)
-@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
 public class MigrateCommandTest {
 
   @AutoClose final PgContainer pgContainer = new PgContainer();

transact-cli/src/test/java/dev/dbos/transact/cli/PgContainer.java

@@ -3,24 +3,86 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Semaphore;
 
 import org.testcontainers.postgresql.PostgreSQLContainer;
 
 public class PgContainer implements AutoCloseable {
-  private final PostgreSQLContainer pgContainer = new PostgreSQLContainer("postgres:18");
+
+  // SIZE should match junit.jupiter.execution.parallel.config.fixed.parallelism value
+  private static final int SIZE = 4;
+  private static final BlockingQueue<PostgreSQLContainer> POOL = new ArrayBlockingQueue<>(SIZE);
+  private static final Semaphore PERMITS = new Semaphore(SIZE);
+
+  static {
+    Runtime.getRuntime()
+        .addShutdownHook(
+            new Thread(
+                () -> {
+                  var containers = new ArrayList<PostgreSQLContainer>();
+                  POOL.drainTo(containers);
+                  containers.forEach(PostgreSQLContainer::stop);
+                }));
+  }
+
+  static PostgreSQLContainer acquire() {
+    try {
+      PERMITS.acquire();
+      var container = POOL.poll();
+      if (container == null) {
+        container = new PostgreSQLContainer("postgres:18");
+        container.start();
+      }
+      return container;
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  static void release(PostgreSQLContainer c) {
+    POOL.offer(c);
+    PERMITS.release();
+  }
+
+  private final PostgreSQLContainer pgContainer;
+  private final String jdbcUrl;
+  private final String dbName;
 
   public PgContainer() {
-    pgContainer.start();
+    // take a container from the pool and create a new database for it
+    pgContainer = acquire();
+    dbName = "test_" + UUID.randomUUID().toString().replace("-", "");
+    jdbcUrl = pgContainer.getJdbcUrl().replaceFirst("/[^/]+$", "/" + dbName);
+
+    try (var conn =
+            DriverManager.getConnection(
+                pgContainer.getJdbcUrl(), pgContainer.getUsername(), pgContainer.getPassword());
+        var stmt = conn.createStatement()) {
+      stmt.execute("CREATE DATABASE " + dbName);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
   }
 
   @Override
   public void close() throws Exception {
-    pgContainer.stop();
+    // drop a database we created and return the container too the pool
+    var _jdbcUrl = pgContainer.getJdbcUrl();
+    try (var conn = DriverManager.getConnection(_jdbcUrl, username(), password());
+        var stmt = conn.createStatement()) {
+      var sql = "DROP DATABASE IF EXISTS %s WITH (FORCE)".formatted(dbName);
+      stmt.execute(sql);
+    }
+    release(pgContainer);
   }
 
   public String jdbcUrl() {
-    return pgContainer.getJdbcUrl();
+    return jdbcUrl;
   }
 
   public String username() {

transact-cli/src/test/resources/junit-platform.properties

@@ -0,0 +1,5 @@
+junit.jupiter.execution.parallel.enabled = true
+junit.jupiter.execution.parallel.mode.default = concurrent
+junit.jupiter.execution.parallel.mode.classes.default = concurrent
+junit.jupiter.execution.parallel.config.strategy = fixed
+junit.jupiter.execution.parallel.config.fixed.parallelism = 4

transact/build.gradle.kts

@@ -57,34 +57,6 @@ tasks.processResources {
   filesMatching("**/app.properties") { expand(mapOf("projectVersion" to projectVersion)) }
 }
 
-tasks.test {
-  useJUnitPlatform()
-  testLogging {
-    events("passed", "skipped", "failed")
-    showStandardStreams = true
-  }
-
-  addTestListener(
-    object : TestListener {
-      override fun beforeSuite(suite: TestDescriptor) {}
-
-      override fun beforeTest(testDescriptor: TestDescriptor) {}
-
-      override fun afterTest(testDescriptor: TestDescriptor, result: TestResult) {}
-
-      override fun afterSuite(suite: TestDescriptor, result: TestResult) {
-        if (suite.parent == null) {
-          println("\nTest Results:")
-          println("  Tests run: ${result.testCount}")
-          println("  Passed: ${result.successfulTestCount}")
-          println("  Failed: ${result.failedTestCount}")
-          println("  Skipped: ${result.skippedTestCount}")
-        }
-      }
-    }
-  )
-}
-
 tasks.withType<KotlinCompile>().configureEach {
   compilerOptions {
     // jvmTarget now uses the JvmTarget enum instead of a String

transact/src/test/java/dev/dbos/transact/admin/AdminServerTest.java

@@ -38,7 +38,6 @@
 import org.mockito.ArgumentCaptor;
 
 @org.junit.jupiter.api.Timeout(value = 2, unit = java.util.concurrent.TimeUnit.MINUTES)
-@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
 class AdminServerTest {
 
   int port;

transact/src/test/java/dev/dbos/transact/client/ClientTest.java

@@ -21,9 +21,9 @@
 
 import com.zaxxer.hikari.HikariDataSource;
 import org.junit.jupiter.api.*;
+import org.junitpioneer.jupiter.RetryingTest;
 
 @org.junit.jupiter.api.Timeout(value = 2, unit = java.util.concurrent.TimeUnit.MINUTES)
-@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
 public class ClientTest {
 
   @AutoClose final PgContainer pgContainer = new PgContainer();
@@ -104,7 +104,7 @@ public void clientSend() throws Exception {
     assertEquals("42-test.message", handle.getResult());
   }
 
-  @Test
+  @RetryingTest(3)
   public void clientEnqueueTimeouts() throws Exception {
     try (var client = pgContainer.dbosClient()) {
       var options = new DBOSClient.EnqueueOptions("ClientServiceImpl", "sleep", "testQueue");

transact/src/test/java/dev/dbos/transact/client/EnqueueOptionsTest.java

@@ -10,7 +10,6 @@
 import org.junit.jupiter.api.Test;
 
 @org.junit.jupiter.api.Timeout(value = 2, unit = java.util.concurrent.TimeUnit.MINUTES)
-@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
 public class EnqueueOptionsTest {
   @Test
   public void enqueueOptionsValidation() throws Exception {

transact/src/test/java/dev/dbos/transact/client/PgSqlClientTest.java

@@ -32,7 +32,6 @@
 import org.postgresql.util.PSQLException;
 
 @org.junit.jupiter.api.Timeout(value = 2, unit = java.util.concurrent.TimeUnit.MINUTES)
-@org.junit.jupiter.api.parallel.Execution(org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT)
 public class PgSqlClientTest {
 
   private static final ObjectMapper MAPPER = new ObjectMapper();