Skip to content

Commit 11bc4d1

Browse files
dweissdweiss
committed
Add more tests, cleanups.
1 parent 8a0664c commit 11bc4d1

10 files changed

Lines changed: 212 additions & 108 deletions

File tree

randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RandomizedTestEngine.java renamed to randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RepeatExecutionTestEngine.java

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,18 @@
1111
import org.junit.platform.engine.support.descriptor.EngineDescriptor;
1212

1313
/**
14-
* A {@link TestEngine} that delegates to JUnit Jupiter and multiplies test execution by running
15-
* tests in multiple top-level iteration containers, each receiving independently derived random
16-
* seeds.
14+
* An experimental {@link TestEngine} that delegates to JUnit Jupiter and multiplies test execution
15+
* by re-running tests in multiple top-level jupiter engines.
1716
*
18-
* <p>The number of iterations is controlled by the {@value #ITERATIONS_PROPERTY} configuration
19-
* parameter (default: {@code 1}). Each iteration's tests have unique context identifiers, causing
20-
* {@link com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension} to derive
21-
* different seeds per iteration even when a fixed root seed is set via {@link SysProps#TESTS_SEED}.
17+
* <p><strong>This is an experimental class and an experimental implementation.</strong>
18+
*
19+
* <p>The number of iterations is controlled by the {@link SysProps#TESTS_ITERS} configuration
20+
* parameter. The default value (0) means no test are executed.
2221
*/
23-
public class RandomizedTestEngine implements TestEngine {
22+
public final class RepeatExecutionTestEngine implements TestEngine {
2423
/** The unique engine ID ({@value}). */
2524
public static final String ENGINE_ID = "randomizedtesting-jupiter";
2625

27-
/** Configuration parameter controlling the number of test iterations. Default: {@code 1}. */
28-
public static final String ITERATIONS_PROPERTY = "tests.iters";
29-
3026
private static final String JUPITER_ENGINE_ID = "junit-jupiter";
3127

3228
private final TestEngine jupiterEngine = loadJupiterEngine();
@@ -41,24 +37,36 @@ public TestDescriptor discover(EngineDiscoveryRequest request, UniqueId uniqueId
4137
int iterations =
4238
request
4339
.getConfigurationParameters()
44-
.get(ITERATIONS_PROPERTY)
40+
.get(SysProps.TESTS_ITERS.propertyKey)
4541
.map(Integer::parseInt)
4642
.orElse(0);
4743

48-
var engineDescriptor = new EngineDescriptor(uniqueId, "Randomized Testing");
44+
UniqueId.Segment jupiterRootSegment;
45+
{
46+
var jupiterRootSegments = UniqueId.forEngine(JUPITER_ENGINE_ID).getSegments();
47+
assert jupiterRootSegments.size() == 1;
48+
jupiterRootSegment = jupiterRootSegments.getFirst();
49+
}
50+
51+
var engineDescriptor = new EngineDescriptor(uniqueId, "RandomizedTesting");
4952
for (int i = 1; i <= iterations; i++) {
50-
var iterationUniqueId = uniqueId.append("seed", String.valueOf(i));
51-
var jupiterRootId = iterationUniqueId.append("engine", JUPITER_ENGINE_ID);
52-
var jupiterDescriptor = jupiterEngine.discover(request, jupiterRootId);
53-
var iterationDescriptor = new TopSeedDescriptor(iterationUniqueId, i);
53+
var iterationUniqueId =
54+
uniqueId.append(ReiterationDescriptor.SEGMENT_TYPE, String.valueOf(i));
55+
var jupiterDescriptor =
56+
jupiterEngine.discover(request, iterationUniqueId.append(jupiterRootSegment));
57+
58+
var iterationDescriptor = new ReiterationDescriptor(iterationUniqueId, i);
5459
iterationDescriptor.addChild(jupiterDescriptor);
5560
engineDescriptor.addChild(iterationDescriptor);
5661
}
62+
5763
return engineDescriptor;
5864
}
5965

60-
public static class TopSeedDescriptor extends AbstractTestDescriptor {
61-
public TopSeedDescriptor(UniqueId uniqueId, long iteration) {
66+
public static class ReiterationDescriptor extends AbstractTestDescriptor {
67+
public static final String SEGMENT_TYPE = "reiteration";
68+
69+
public ReiterationDescriptor(UniqueId uniqueId, long iteration) {
6270
super(uniqueId, "Iteration " + iteration);
6371
}
6472

@@ -74,12 +82,13 @@ public void execute(ExecutionRequest request) {
7482
var listener = request.getEngineExecutionListener();
7583
listener.executionStarted(engineDescriptor);
7684
for (var child : engineDescriptor.getChildren()) {
77-
executeIteration((TopSeedDescriptor) child, request);
85+
executeIteration((ReiterationDescriptor) child, request);
7886
}
7987
listener.executionFinished(engineDescriptor, TestExecutionResult.successful());
8088
}
8189

82-
private void executeIteration(TopSeedDescriptor iterationDescriptor, ExecutionRequest request) {
90+
private void executeIteration(
91+
ReiterationDescriptor iterationDescriptor, ExecutionRequest request) {
8392
var listener = request.getEngineExecutionListener();
8493
listener.executionStarted(iterationDescriptor);
8594
for (var jupiterDescriptor : iterationDescriptor.getChildren()) {

‎randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SysProps.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ public enum SysProps {
2121
*/
2222
TESTS_RANDOM_ASSERTING("tests.random.asserting"),
2323

24+
/**
25+
* Test reiteration count for the experimental test engine that re-runs full suites multiple times
26+
* (with a constant or varying seed).
27+
*
28+
* @see RepeatExecutionTestEngine
29+
*/
30+
TESTS_ITERS("tests.iters"),
31+
2432
/**
2533
* A "multiplier" for certain methods that return random values in {@link RandomizedTest}.
2634
*

‎randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/internals/RandomizedContextImpl.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import com.carrotsearch.randomizedtesting.jupiter.FixSeed;
44
import com.carrotsearch.randomizedtesting.jupiter.Hashing;
55
import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
6-
import com.carrotsearch.randomizedtesting.jupiter.RandomizedTestEngine;
6+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
77
import com.carrotsearch.randomizedtesting.jupiter.Seed;
88
import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
99
import com.carrotsearch.randomizedtesting.jupiter.SysProps;
@@ -123,7 +123,8 @@ RandomizedContextImpl deriveNew(ExtensionContext extensionContext) {
123123
if (nextSeed.isUnspecified()) {
124124
var uniqueId = UniqueId.parse(extensionContext.getUniqueId());
125125
var strippedId = uniqueId.toString();
126-
if (Objects.equals(RandomizedTestEngine.ENGINE_ID, uniqueId.getEngineId().orElse(null))) {
126+
if (Objects.equals(
127+
RepeatExecutionTestEngine.ENGINE_ID, uniqueId.getEngineId().orElse(null))) {
127128
var segments = uniqueId.getSegments();
128129
segments = segments.subList(2, segments.size());
129130
var stripped = UniqueId.root(segments.getFirst().getType(), segments.getFirst().getValue());

‎randomizedtesting-jupiter/src/main/java/module-info.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
2+
13
module com.carrotsearch.randomizedtesting {
24
requires org.junit.jupiter.api;
35
requires org.junit.jupiter.params;
@@ -16,5 +18,5 @@
1618
provides org.junit.jupiter.api.extension.Extension with
1719
com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension;
1820
provides org.junit.platform.engine.TestEngine with
19-
com.carrotsearch.randomizedtesting.jupiter.RandomizedTestEngine;
21+
RepeatExecutionTestEngine;
2022
}

‎randomizedtesting-jupiter/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
com.carrotsearch.randomizedtesting.jupiter.RandomizedTestEngine
1+
com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F004_TestEngineIterations.java‎

Lines changed: 0 additions & 82 deletions
This file was deleted.

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package com.carrotsearch.randomizedtesting.tests;
2+
3+
import static com.carrotsearch.randomizedtesting.tests.infra.TestInfra.*;
4+
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
5+
import static org.junit.platform.testkit.engine.EventConditions.event;
6+
import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
7+
import static org.junit.platform.testkit.engine.EventConditions.test;
8+
9+
import com.carrotsearch.randomizedtesting.jupiter.Randomized;
10+
import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
11+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
12+
import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
13+
import com.carrotsearch.randomizedtesting.jupiter.SysProps;
14+
import com.carrotsearch.randomizedtesting.tests.infra.IgnoreInStandaloneRuns;
15+
import com.carrotsearch.randomizedtesting.tests.infra.TestInfra;
16+
import java.io.PrintWriter;
17+
import java.util.List;
18+
import org.assertj.core.api.Assertions;
19+
import org.junit.jupiter.api.Test;
20+
21+
/** Verifies that {@link RepeatExecutionTestEngine} correctly multiplies test execution. */
22+
public class F007_TestReiteration {
23+
@Test
24+
void noReiterationsByDefault() {
25+
var result =
26+
collectExecutionResults(
27+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
28+
.selectors(selectClass(SimpleTest.class)));
29+
30+
result
31+
.results()
32+
.testEvents()
33+
.assertThatEvents()
34+
.doNotHave(event(finishedWithFailure()))
35+
.haveExactly(0, event(test()));
36+
}
37+
38+
@Test
39+
void testsAreMultipliedByIterationCount() {
40+
var result =
41+
collectExecutionResults(
42+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
43+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "3")
44+
.selectors(selectClass(SimpleTest.class)));
45+
46+
result.results().testEvents().assertStatistics(s -> s.finished(3).succeeded(3));
47+
}
48+
49+
@Test
50+
void seedsAreIdenticalAcrossIterationsWithFixedRootSeed() {
51+
var iterations = 5;
52+
var result =
53+
collectExecutionResults(
54+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
55+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
56+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
57+
.selectors(selectClass(SimpleTest.class)));
58+
59+
result
60+
.results()
61+
.testEvents()
62+
.assertStatistics(s -> s.finished(iterations).succeeded(iterations));
63+
64+
// Each iteration should have the same seed chain because we strip the top-level reiteration
65+
// segments.
66+
Assertions.assertThat(result.capturedOutput().values().stream().distinct())
67+
.as("seed chains should be the same across iterations")
68+
.hasSize(1);
69+
70+
Assertions.assertThat(
71+
result.capturedOutput().values().stream()
72+
.map(value -> value.split("\\s")[0])
73+
.map(v -> SeedChain.parse(v))
74+
.map(chain -> chain.seeds().getFirst().toString()))
75+
.allMatch(v -> v.equals("DEADBEEF"));
76+
}
77+
78+
@Test
79+
void seedsAreDifferentAcrossIterationsWithNoRootSeed() {
80+
var iterations = 5;
81+
var result =
82+
collectExecutionResults(
83+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
84+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
85+
.selectors(selectClass(SimpleTest.class)));
86+
87+
result
88+
.results()
89+
.testEvents()
90+
.assertStatistics(s -> s.finished(iterations).succeeded(iterations));
91+
92+
// Each iteration should have a random root seed if there is no top-level fixed seed.
93+
Assertions.assertThat(result.capturedOutput().values().stream().distinct())
94+
.as("seed chains should be different across iterations")
95+
.hasSizeBetween(iterations - 2, iterations);
96+
}
97+
98+
@Test
99+
void randomnessIsIdenticalForJupiterAndReiteratedTests() {
100+
List<String> jupiterResults;
101+
List<String> repeatedExecutionResults;
102+
103+
{
104+
var result =
105+
collectExecutionResults(
106+
TestInfra.testKitBuilder()
107+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
108+
.selectors(selectClass(SimpleTest.class)));
109+
110+
result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
111+
112+
jupiterResults = result.capturedOutput().values().stream().toList();
113+
}
114+
115+
{
116+
var result =
117+
collectExecutionResults(
118+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
119+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "1")
120+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
121+
.selectors(selectClass(SimpleTest.class)));
122+
123+
result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
124+
125+
repeatedExecutionResults = result.capturedOutput().values().stream().toList();
126+
}
127+
128+
Assertions.assertThat(jupiterResults).containsExactlyElementsOf(repeatedExecutionResults);
129+
}
130+
131+
@Randomized
132+
static class SimpleTest extends IgnoreInStandaloneRuns {
133+
@Test
134+
void test(PrintWriter pw, RandomizedContext ctx) {
135+
pw.println(ctx.getSeedChain() + " " + ctx.getRandom().nextLong());
136+
}
137+
}
138+
}

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Feature: re-run (reiterate) one or more tests multiple times, with constant or varying seeds.
2+
3+
## Functionality
4+
5+
* It should be possible to "rerun" one or more tests multiple times with the same or varying seed to check if a
6+
problematic
7+
failure depends on the seed (reproduces) or if it's not reproducible.
8+
9+
* The reiteration is controlled by a system property `tests.iters`, taking the number of reiterations to execute.
10+
11+
* Only junit jupiter tests are reiterated at the moment (by wrapping jupiter test engine and delegating execution to
12+
it).
13+
14+
* If `tests.seed` is fixed, all reiterations should result in the same randomness and execution results. Otherwise,
15+
each reiteration starts with a random root seed.
16+
17+
## Migration notes (from randomizedtesting for junit4)
18+
19+
* Test reiteration will run the full stack of all extensions and hooks - it's not a mere re-execution of an
20+
individual test.

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/experiments/AdHoc.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
import com.carrotsearch.randomizedtesting.jupiter.Randomized;
44
import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
55
import org.junit.jupiter.api.BeforeAll;
6+
import org.junit.jupiter.api.Disabled;
67
import org.junit.jupiter.api.RepeatedTest;
78

89
public class AdHoc {
9-
// @Disabled
10+
@Disabled
1011
@Randomized
1112
public static class TestClass {
1213
@BeforeAll

0 commit comments

Comments
 (0)