Add support for tests.iters (reiteration of tests)

randomizedtesting-jupiter/build.gradle

@@ -1,14 +1,15 @@
 dependencies {
     api libs.junit.jupiter
+    implementation libs.junit.jupiter.engine
 
     testImplementation libs.assertj
-    testImplementation libs.junit.jupiter.engine
     testImplementation libs.junit.platform.testkit
 }
 
 test {
     useJUnitPlatform {
         excludeTags 'nested-integration-test'
+        includeEngines 'randomizedtesting-jupiter'
     }
     jvmArgs("-Dnet.bytebuddy.safe=true")
 }

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RepeatExecutionTestEngine.java

@@ -0,0 +1,117 @@
+package com.carrotsearch.randomizedtesting.jupiter;
+
+import java.util.ServiceLoader;
+import org.junit.platform.engine.EngineDiscoveryRequest;
+import org.junit.platform.engine.ExecutionRequest;
+import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestEngine;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
+import org.junit.platform.engine.support.descriptor.EngineDescriptor;
+
+/**
+ * An experimental {@link TestEngine} that delegates to JUnit Jupiter and multiplies test execution
+ * by re-running tests in multiple top-level jupiter engines.
+ *
+ * <p><strong>This is an experimental class and an experimental implementation.</strong>
+ *
+ * <p>The number of iterations is controlled by the {@link SysProps#TESTS_ITERS} configuration
+ * parameter. The default value (0) means no test are executed.
+ */
+public final class RepeatExecutionTestEngine implements TestEngine {
+  /** The unique engine ID ({@value}). */
+  public static final String ENGINE_ID = "randomizedtesting-jupiter";
+
+  private static final String JUPITER_ENGINE_ID = "junit-jupiter";
+
+  private final TestEngine jupiterEngine = loadJupiterEngine();
+
+  @Override
+  public String getId() {
+    return ENGINE_ID;
+  }
+
+  @Override
+  public TestDescriptor discover(EngineDiscoveryRequest request, UniqueId uniqueId) {
+    int iterations =
+        request
+            .getConfigurationParameters()
+            .get(SysProps.TESTS_ITERS.propertyKey)
+            .map(Integer::parseInt)
+            .orElse(0);
+
+    UniqueId.Segment jupiterRootSegment;
+    {
+      var jupiterRootSegments = UniqueId.forEngine(JUPITER_ENGINE_ID).getSegments();
+      assert jupiterRootSegments.size() == 1;
+      jupiterRootSegment = jupiterRootSegments.getFirst();
+    }
+
+    var engineDescriptor = new EngineDescriptor(uniqueId, "RandomizedTesting");
+    for (int i = 1; i <= iterations; i++) {
+      var iterationUniqueId =
+          uniqueId.append(ReiterationDescriptor.SEGMENT_TYPE, String.valueOf(i));
+      var jupiterDescriptor =
+          jupiterEngine.discover(request, iterationUniqueId.append(jupiterRootSegment));
+
+      var iterationDescriptor = new ReiterationDescriptor(iterationUniqueId, i);
+      iterationDescriptor.addChild(jupiterDescriptor);
+      engineDescriptor.addChild(iterationDescriptor);
+    }
+
+    return engineDescriptor;
+  }
+
+  public static class ReiterationDescriptor extends AbstractTestDescriptor {
+    public static final String SEGMENT_TYPE = "reiteration";
+
+    public ReiterationDescriptor(UniqueId uniqueId, long iteration) {
+      super(uniqueId, "Iteration " + iteration);
+    }
+
+    @Override
+    public Type getType() {
+      return Type.CONTAINER;
+    }
+  }
+
+  @Override
+  public void execute(ExecutionRequest request) {
+    var engineDescriptor = request.getRootTestDescriptor();
+    var listener = request.getEngineExecutionListener();
+    listener.executionStarted(engineDescriptor);
+    for (var child : engineDescriptor.getChildren()) {
+      executeIteration((ReiterationDescriptor) child, request);
+    }
+    listener.executionFinished(engineDescriptor, TestExecutionResult.successful());
+  }
+
+  private void executeIteration(
+      ReiterationDescriptor iterationDescriptor, ExecutionRequest request) {
+    var listener = request.getEngineExecutionListener();
+    listener.executionStarted(iterationDescriptor);
+    for (var jupiterDescriptor : iterationDescriptor.getChildren()) {
+      jupiterEngine.execute(
+          ExecutionRequest.create(
+              jupiterDescriptor,
+              listener,
+              request.getConfigurationParameters(),
+              request.getOutputDirectoryCreator(),
+              request.getStore(),
+              request.getCancellationToken()));
+    }
+    listener.executionFinished(iterationDescriptor, TestExecutionResult.successful());
+  }
+
+  private static TestEngine loadJupiterEngine() {
+    return ServiceLoader.load(TestEngine.class).stream()
+        .filter(p -> p.type().getName().equals("org.junit.jupiter.engine.JupiterTestEngine"))
+        .map(ServiceLoader.Provider::get)
+        .findFirst()
+        .orElseThrow(
+            () ->
+                new IllegalStateException(
+                    "JUnit Jupiter engine not found; add junit-jupiter-engine to the classpath"));
+  }
+}

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SysProps.java

@@ -21,6 +21,14 @@ public enum SysProps {
    */
   TESTS_RANDOM_ASSERTING("tests.random.asserting"),
 
+  /**
+   * Test reiteration count for the experimental test engine that re-runs full suites multiple times
+   * (with a constant or varying seed).
+   *
+   * @see RepeatExecutionTestEngine
+   */
+  TESTS_ITERS("tests.iters"),
+
   /**
    * A "multiplier" for certain methods that return random values in {@link RandomizedTest}.
    *

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/internals/RandomizedContextImpl.java

@@ -3,6 +3,7 @@
 import com.carrotsearch.randomizedtesting.jupiter.FixSeed;
 import com.carrotsearch.randomizedtesting.jupiter.Hashing;
 import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
+import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
 import com.carrotsearch.randomizedtesting.jupiter.Seed;
 import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
 import com.carrotsearch.randomizedtesting.jupiter.SysProps;
@@ -15,6 +16,7 @@
 import java.util.Random;
 import java.util.function.LongFunction;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.platform.engine.UniqueId;
 
 public final class RandomizedContextImpl implements Closeable, RandomizedContext {
   private final RandomizedContextImpl parent;
@@ -106,7 +108,7 @@ RandomizedContextImpl deriveNew(ExtensionContext extensionContext) {
           throw new RuntimeException(
               String.format(
                   Locale.ROOT,
-                  "@%s annotatoin must declare concrete seeds or seed chains on: %s",
+                  "@%s annotation must declare concrete seeds or seed chains on: %s",
                   FixSeed.class.getName(),
                   extensionContext.getElement().get()));
         }
@@ -119,7 +121,19 @@ RandomizedContextImpl deriveNew(ExtensionContext extensionContext) {
     var nextSeed = firstAndRest.first();
     var remainingChain = firstAndRest.rest();
     if (nextSeed.isUnspecified()) {
-      nextSeed = new Seed(this.seed.value() ^ Hashing.hash(extensionContext.getUniqueId()));
+      var uniqueId = UniqueId.parse(extensionContext.getUniqueId());
+      var strippedId = uniqueId.toString();
+      if (Objects.equals(
+          RepeatExecutionTestEngine.ENGINE_ID, uniqueId.getEngineId().orElse(null))) {
+        var segments = uniqueId.getSegments();
+        segments = segments.subList(2, segments.size());
+        var stripped = UniqueId.root(segments.getFirst().getType(), segments.getFirst().getValue());
+        for (int i = 1; i < segments.size(); i++) {
+          stripped = stripped.append(segments.get(i));
+        }
+        strippedId = stripped.toString();
+      }
+      nextSeed = new Seed(this.seed.value() ^ Hashing.hash(strippedId));
     }
 
     return new RandomizedContextImpl(

randomizedtesting-jupiter/src/main/java/module-info.java

@@ -1,4 +1,4 @@
-import com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension;
+import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
 
 module com.carrotsearch.randomizedtesting {
   requires org.junit.jupiter.api;
@@ -13,5 +13,13 @@
       org.junit.platform.commons;
 
   provides org.junit.jupiter.api.extension.Extension with
-      RandomizedContextExtension;
+      com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension;
+
+  // These entries install support for tests.iters (RepeatedExecutionTestEngine).
+  requires org.junit.platform.engine;
+
+  uses org.junit.platform.engine.TestEngine;
+
+  provides org.junit.platform.engine.TestEngine with
+      RepeatExecutionTestEngine;
 }

randomizedtesting-jupiter/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -0,0 +1 @@
+com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine

randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.java

@@ -0,0 +1,138 @@
+package com.carrotsearch.randomizedtesting.tests;
+
+import static com.carrotsearch.randomizedtesting.tests.infra.TestInfra.*;
+import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
+import static org.junit.platform.testkit.engine.EventConditions.event;
+import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
+import static org.junit.platform.testkit.engine.EventConditions.test;
+
+import com.carrotsearch.randomizedtesting.jupiter.Randomized;
+import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
+import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
+import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
+import com.carrotsearch.randomizedtesting.jupiter.SysProps;
+import com.carrotsearch.randomizedtesting.tests.infra.IgnoreInStandaloneRuns;
+import com.carrotsearch.randomizedtesting.tests.infra.TestInfra;
+import java.io.PrintWriter;
+import java.util.List;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/** Verifies that {@link RepeatExecutionTestEngine} correctly multiplies test execution. */
+public class F007_TestReiteration {
+  @Test
+  void noReiterationsByDefault() {
+    var result =
+        collectExecutionResults(
+            TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
+                .selectors(selectClass(SimpleTest.class)));
+
+    result
+        .results()
+        .testEvents()
+        .assertThatEvents()
+        .doNotHave(event(finishedWithFailure()))
+        .haveExactly(0, event(test()));
+  }
+
+  @Test
+  void testsAreMultipliedByIterationCount() {
+    var result =
+        collectExecutionResults(
+            TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
+                .configurationParameter(SysProps.TESTS_ITERS.propertyKey, "3")
+                .selectors(selectClass(SimpleTest.class)));
+
+    result.results().testEvents().assertStatistics(s -> s.finished(3).succeeded(3));
+  }
+
+  @Test
+  void seedsAreIdenticalAcrossIterationsWithFixedRootSeed() {
+    var iterations = 5;
+    var result =
+        collectExecutionResults(
+            TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
+                .configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
+                .configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
+                .selectors(selectClass(SimpleTest.class)));
+
+    result
+        .results()
+        .testEvents()
+        .assertStatistics(s -> s.finished(iterations).succeeded(iterations));
+
+    // Each iteration should have the same seed chain because we strip the top-level reiteration
+    // segments.
+    Assertions.assertThat(result.capturedOutput().values().stream().distinct())
+        .as("seed chains should be the same across iterations")
+        .hasSize(1);
+
+    Assertions.assertThat(
+            result.capturedOutput().values().stream()
+                .map(value -> value.split("\\s")[0])
+                .map(v -> SeedChain.parse(v))
+                .map(chain -> chain.seeds().getFirst().toString()))
+        .allMatch(v -> v.equals("DEADBEEF"));
+  }
+
+  @Test
+  void seedsAreDifferentAcrossIterationsWithNoRootSeed() {
+    var iterations = 5;
+    var result =
+        collectExecutionResults(
+            TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
+                .configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
+                .selectors(selectClass(SimpleTest.class)));
+
+    result
+        .results()
+        .testEvents()
+        .assertStatistics(s -> s.finished(iterations).succeeded(iterations));
+
+    // Each iteration should have a random root seed if there is no top-level fixed seed.
+    Assertions.assertThat(result.capturedOutput().values().stream().distinct())
+        .as("seed chains should be different across iterations")
+        .hasSizeBetween(iterations - 2, iterations);
+  }
+
+  @Test
+  void randomnessIsIdenticalForJupiterAndReiteratedTests() {
+    List<String> jupiterResults;
+    List<String> repeatedExecutionResults;
+
+    {
+      var result =
+          collectExecutionResults(
+              TestInfra.testKitBuilder()
+                  .configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
+                  .selectors(selectClass(SimpleTest.class)));
+
+      result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
+
+      jupiterResults = result.capturedOutput().values().stream().toList();
+    }
+
+    {
+      var result =
+          collectExecutionResults(
+              TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
+                  .configurationParameter(SysProps.TESTS_ITERS.propertyKey, "1")
+                  .configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
+                  .selectors(selectClass(SimpleTest.class)));
+
+      result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
+
+      repeatedExecutionResults = result.capturedOutput().values().stream().toList();
+    }
+
+    Assertions.assertThat(jupiterResults).containsExactlyElementsOf(repeatedExecutionResults);
+  }
+
+  @Randomized
+  static class SimpleTest extends IgnoreInStandaloneRuns {
+    @Test
+    void test(PrintWriter pw, RandomizedContext ctx) {
+      pw.println(ctx.getSeedChain() + " " + ctx.getRandom().nextLong());
+    }
+  }
+}

randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.md

@@ -0,0 +1,20 @@
+# Feature: re-run (reiterate) one or more tests multiple times, with constant or varying seeds.
+
+## Functionality
+
+* It should be possible to "rerun" one or more tests multiple times with the same or varying seed to check if a
+  problematic
+  failure depends on the seed (reproduces) or if it's not reproducible.
+
+* The reiteration is controlled by a system property `tests.iters`, taking the number of reiterations to execute.
+
+* Only junit jupiter tests are reiterated at the moment (by wrapping jupiter test engine and delegating execution to
+  it).
+
+* If `tests.seed` is fixed, all reiterations should result in the same randomness and execution results. Otherwise,
+  each reiteration starts with a random root seed.
+
+## Migration notes (from randomizedtesting for junit4)
+
+* Test reiteration will run the full stack of all extensions and hooks - it's not a mere re-execution of an
+  individual test.