Merge pull request #4 from randomizedtesting/fix-seed-annotation

Add FixSeed annotation

Author: dweiss

etc/junit4-missing-features.txt

@@ -1,6 +1,6 @@
 [ai generated overview of junit4 features]
 
-4. Shuffled test execution order and seed annotations
+4. seed annotations
    - @Seed on a class fixes the main seed, making execution fully deterministic
    - @Seeds / @Seed on a method pins a per-method seed for regression coverage
      while still running once with a fresh random seed
@@ -44,3 +44,9 @@
    - Utility methods on RandomizedTest: randomInt(), randomIntBetween(),
      randomBoolean(), randomFloat(), etc.
    - Encourages testing over a broad input domain rather than fixed values
+
+[possibly doable with a custom test engine]
+
+- predictably shuffled test execution order
+- blowing up test reps using tests.iters
+-

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/FixSeed.java

@@ -0,0 +1,24 @@
+package com.carrotsearch.randomizedtesting.jupiter;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * This annotation should be placed on classes or methods that are {@link Randomized} and would like
+ * to use a constant seed (for reproducing a problem or other reasons).
+ *
+ * <p>Note that seed fixing is always possible by setting {@link
+ * com.carrotsearch.randomizedtesting.jupiter.RandomizedContextSupplier.SysProps#TESTS_SEED} system
+ * property, this is just convenience.
+ */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(RandomizedContextSupplier.class)
+public @interface FixSeed {
+  String value();
+}

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedContext.java

@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Random;
 import org.junit.jupiter.api.extension.ExtensionContext;
@@ -76,14 +77,33 @@ RandomizedContext deriveNew(ExtensionContext extensionContext) {
       }
     }
 
-    var firstAndRest = this.remainingSeedChain.pop();
+    SeedChain seedChain;
+    var annotationSeed = extensionContext.getElement().map(e -> e.getAnnotation(FixSeed.class));
+    if (annotationSeed.isPresent()) {
+      seedChain = SeedChain.parse(annotationSeed.get().value());
+      for (var seed : seedChain.seeds()) {
+        if (seed.isUnspecified()) {
+          throw new RuntimeException(
+              String.format(
+                  Locale.ROOT,
+                  "@%s annotatoin must declare concrete seeds or seed chains on: %s",
+                  FixSeed.class.getName(),
+                  extensionContext.getElement().get()));
+        }
+      }
+    } else {
+      seedChain = this.remainingSeedChain;
+    }
+
+    var firstAndRest = seedChain.pop();
     var nextSeed = firstAndRest.first();
+    var remainingChain = firstAndRest.rest();
     if (nextSeed.isUnspecified()) {
       nextSeed = new Seed(this.seed.value() ^ Hashing.longHash(extensionContext.getUniqueId()));
     }
 
     return new RandomizedContext(
-        extensionContext.getUniqueId(), this, randomFactory, nextSeed, firstAndRest.rest());
+        extensionContext.getUniqueId(), this, randomFactory, nextSeed, remainingChain);
   }
 
   @Override

randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.java

@@ -0,0 +1,98 @@
+package com.carrotsearch.randomizedtesting.jupiter;
+
+import static com.carrotsearch.randomizedtesting.jupiter.infra.TestInfra.*;
+import static org.junit.platform.testkit.engine.EventConditions.*;
+
+import com.carrotsearch.randomizedtesting.jupiter.infra.IgnoreInStandaloneRuns;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.RepeatedTest;
+import org.junit.jupiter.api.Test;
+
+/** Ensure there is a way to quickly "fix" the seed for the given method or class. */
+public class F004_SeedFixing {
+  @Nested
+  class TestSeedAnnotation {
+    @Test
+    public void testSeedFixing() {
+      collectExecutionResults(
+              testKitBuilder(T1.class)
+                  .configurationParameter(
+                      RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead:beef:cafe"))
+          .results()
+          .allEvents()
+          .assertThatEvents()
+          .doNotHave(event(finishedWithFailure()));
+    }
+
+    @Randomized
+    static class T1 extends IgnoreInStandaloneRuns {
+      @Test
+      @FixSeed("babe")
+      void simpleTest(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).isEqualTo("[DEAD:BEEF:BABE]");
+      }
+    }
+
+    @Test
+    public void testClassSeedFixing() {
+      collectExecutionResults(
+              testKitBuilder(T2.class)
+                  .configurationParameter(
+                      RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead"))
+          .results()
+          .allEvents()
+          .assertThatEvents()
+          .doNotHave(event(finishedWithFailure()));
+    }
+
+    @Randomized
+    @FixSeed("babe")
+    static class T2 extends IgnoreInStandaloneRuns {
+      @Test
+      void ta(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:");
+      }
+
+      @Test
+      void tb(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:");
+      }
+    }
+
+    @Randomized
+    @FixSeed("babe:caca")
+    static class T3 extends IgnoreInStandaloneRuns {
+      @Test
+      void ta(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:CACA]");
+      }
+
+      @Test
+      void tb(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).startsWith("[DEAD:BABE:CACA]");
+      }
+    }
+
+    @Test
+    public void testRepeatedTests() {
+      collectExecutionResults(
+              testKitBuilder(T4.class)
+                  .configurationParameter(
+                      RandomizedContextSupplier.SysProps.TESTS_SEED.propertyKey, "dead"))
+          .results()
+          .allEvents()
+          .assertThatEvents()
+          .doNotHave(event(finishedWithFailure()));
+    }
+
+    @Randomized
+    static class T4 extends IgnoreInStandaloneRuns {
+      @RepeatedTest(5)
+      @FixSeed("babe")
+      void simpleTest(RandomizedContext ctx) {
+        Assertions.assertThat(ctx.getSeedChain().toString()).endsWith(":BABE]");
+      }
+    }
+  }
+}

randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/F004_SeedFixing.md

@@ -0,0 +1,80 @@
+# Feature: seed fixing using `FixSeed` annotation
+
+## Functionality
+
+* It should be possible to "fix" (make constant, regardless of the
+  randomization state) the random seed for methods and classes. This can be achieved using `@FixSeed` annotation
+
+```java
+
+@Randomized
+@FixSeed("cafebabe")
+public class TestClass {
+    @Test
+    public void testMethod(Random ctx) {
+    }
+}
+```
+
+it is also possible to fix the seed for a particular method, although this allows state randomization at
+class level:
+
+```java
+
+@Randomized
+public class TestClass {
+    @Test
+    @FixSeed("cafebabe")
+    public void testMethod(Random ctx) {
+    }
+}
+```
+
+* The value of the `@FixSeed` annotation can be a single seed or a chain of seeds, affecting nested contexts.
+
+```java
+
+@Randomized
+@FixSeed("cafebabe:deadbeef")
+public class TestClass {
+    @Test
+    public void testMethod(Random ctx) {
+    }
+}
+```
+
+* `@FixSeed` can be used to rerun the same tests multiple times with a constant seed or predictably varying seed. For
+  example, this test runs 5 times with the same seed/ randomness at the test level:
+
+```java
+
+@Randomized
+public class TestClass {
+    @RepeatedTest(5)
+    @FixSeed("babe")
+    public void testMethod(Random ctx) {
+    }
+}
+```
+
+but this test runs 5 times, each time with a different (but predictable, derived from the parent) seed:
+
+```java
+
+@Randomized
+@FixSeed("babe")
+public class TestClass {
+    @RepeatedTest(5)
+    public void testMethod(Random ctx) {
+    }
+}
+```
+
+## Migration notes (from randomizedtesting for junit4)
+
+* `@FixSeed` is renamed from the `@Seed` annotation, used previously.
+
+* There is no way to provide multiple seeds (`@Seeds` annotation), there is no replacement for this functionality.
+
+* There are subtle differences in how the annotation propagates but overall think of the seed annotation placement
+(method, class) as affecting the corresponding JUnit5 extension context (its path in the test's UniqueId).

randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/jupiter/experiments/JupiterCallbackMethodOrder.java

@@ -33,37 +33,41 @@
 @ValueSource(strings = {"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
 public class JupiterCallbackMethodOrder {
   static {
-    System.out.println("Static constructor.");
+    System.out.println("tclass: static constructor.");
   }
 
   public JupiterCallbackMethodOrder(String param) {
-    System.out.println("constructor: " + param);
+    System.out.println("tclass: constructor: " + param);
   }
 
   @BeforeAll
   public static void beforeAll() {
-    System.out.println("Before all.");
+    System.out.println("tclass: beforeAll.");
   }
 
   @AfterAll
   public static void afterAll() {
-    System.out.println("After all.");
+    System.out.println("tclass: afterAll.");
   }
 
   @BeforeEach
-  public void before() {}
+  public void before() {
+    System.out.println("tclass: beforeEach.");
+  }
 
   @AfterEach
-  public void after() {}
+  public void after() {
+    System.out.println("tclass: afterEach.");
+  }
 
   @Test
   public void b() {
-    System.out.println("Test b.");
+    System.out.println("tclass: test b.");
   }
 
   @Test
   public void a() {
-    System.out.println("Test a.");
+    System.out.println("tclass: test a.");
   }
 
   public static class DebugExt
@@ -80,7 +84,8 @@ public static class DebugExt
           InvocationInterceptor {
 
     private static void log(String callback, ExtensionContext context) {
-      System.out.println(callback + " | " + context.getUniqueId());
+      System.out.println(
+          "ext: " + callback + "\n  " + context.getUniqueId() + "\n  " + context.getElement());
     }
 
     @Override