Skip to content

Commit bacb27a

Browse files
dweissdweiss
authored
Add support for tests.iters (reiteration of tests) (#20)
* A conceptual test engine duplicating tests above jupiter. * Replace top level iteration class. * Add more tests, cleanups. * Cleanups.
1 parent 3c035de commit bacb27a

10 files changed

Lines changed: 326 additions & 6 deletions

File tree

‎randomizedtesting-jupiter/build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
dependencies {
22
api libs.junit.jupiter
3+
implementation libs.junit.jupiter.engine
34

45
testImplementation libs.assertj
5-
testImplementation libs.junit.jupiter.engine
66
testImplementation libs.junit.platform.testkit
77
}
88

99
test {
1010
useJUnitPlatform {
1111
excludeTags 'nested-integration-test'
12+
includeEngines 'randomizedtesting-jupiter'
1213
}
1314
jvmArgs("-Dnet.bytebuddy.safe=true")
1415
}

‎randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/RepeatExecutionTestEngine.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package com.carrotsearch.randomizedtesting.jupiter;
2+
3+
import java.util.ServiceLoader;
4+
import org.junit.platform.engine.EngineDiscoveryRequest;
5+
import org.junit.platform.engine.ExecutionRequest;
6+
import org.junit.platform.engine.TestDescriptor;
7+
import org.junit.platform.engine.TestEngine;
8+
import org.junit.platform.engine.TestExecutionResult;
9+
import org.junit.platform.engine.UniqueId;
10+
import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
11+
import org.junit.platform.engine.support.descriptor.EngineDescriptor;
12+
13+
/**
14+
* An experimental {@link TestEngine} that delegates to JUnit Jupiter and multiplies test execution
15+
* by re-running tests in multiple top-level jupiter engines.
16+
*
17+
* <p><strong>This is an experimental class and an experimental implementation.</strong>
18+
*
19+
* <p>The number of iterations is controlled by the {@link SysProps#TESTS_ITERS} configuration
20+
* parameter. The default value (0) means no test are executed.
21+
*/
22+
public final class RepeatExecutionTestEngine implements TestEngine {
23+
/** The unique engine ID ({@value}). */
24+
public static final String ENGINE_ID = "randomizedtesting-jupiter";
25+
26+
private static final String JUPITER_ENGINE_ID = "junit-jupiter";
27+
28+
private final TestEngine jupiterEngine = loadJupiterEngine();
29+
30+
@Override
31+
public String getId() {
32+
return ENGINE_ID;
33+
}
34+
35+
@Override
36+
public TestDescriptor discover(EngineDiscoveryRequest request, UniqueId uniqueId) {
37+
int iterations =
38+
request
39+
.getConfigurationParameters()
40+
.get(SysProps.TESTS_ITERS.propertyKey)
41+
.map(Integer::parseInt)
42+
.orElse(0);
43+
44+
UniqueId.Segment jupiterRootSegment;
45+
{
46+
var jupiterRootSegments = UniqueId.forEngine(JUPITER_ENGINE_ID).getSegments();
47+
assert jupiterRootSegments.size() == 1;
48+
jupiterRootSegment = jupiterRootSegments.getFirst();
49+
}
50+
51+
var engineDescriptor = new EngineDescriptor(uniqueId, "RandomizedTesting");
52+
for (int i = 1; i <= iterations; i++) {
53+
var iterationUniqueId =
54+
uniqueId.append(ReiterationDescriptor.SEGMENT_TYPE, String.valueOf(i));
55+
var jupiterDescriptor =
56+
jupiterEngine.discover(request, iterationUniqueId.append(jupiterRootSegment));
57+
58+
var iterationDescriptor = new ReiterationDescriptor(iterationUniqueId, i);
59+
iterationDescriptor.addChild(jupiterDescriptor);
60+
engineDescriptor.addChild(iterationDescriptor);
61+
}
62+
63+
return engineDescriptor;
64+
}
65+
66+
public static class ReiterationDescriptor extends AbstractTestDescriptor {
67+
public static final String SEGMENT_TYPE = "reiteration";
68+
69+
public ReiterationDescriptor(UniqueId uniqueId, long iteration) {
70+
super(uniqueId, "Iteration " + iteration);
71+
}
72+
73+
@Override
74+
public Type getType() {
75+
return Type.CONTAINER;
76+
}
77+
}
78+
79+
@Override
80+
public void execute(ExecutionRequest request) {
81+
var engineDescriptor = request.getRootTestDescriptor();
82+
var listener = request.getEngineExecutionListener();
83+
listener.executionStarted(engineDescriptor);
84+
for (var child : engineDescriptor.getChildren()) {
85+
executeIteration((ReiterationDescriptor) child, request);
86+
}
87+
listener.executionFinished(engineDescriptor, TestExecutionResult.successful());
88+
}
89+
90+
private void executeIteration(
91+
ReiterationDescriptor iterationDescriptor, ExecutionRequest request) {
92+
var listener = request.getEngineExecutionListener();
93+
listener.executionStarted(iterationDescriptor);
94+
for (var jupiterDescriptor : iterationDescriptor.getChildren()) {
95+
jupiterEngine.execute(
96+
ExecutionRequest.create(
97+
jupiterDescriptor,
98+
listener,
99+
request.getConfigurationParameters(),
100+
request.getOutputDirectoryCreator(),
101+
request.getStore(),
102+
request.getCancellationToken()));
103+
}
104+
listener.executionFinished(iterationDescriptor, TestExecutionResult.successful());
105+
}
106+
107+
private static TestEngine loadJupiterEngine() {
108+
return ServiceLoader.load(TestEngine.class).stream()
109+
.filter(p -> p.type().getName().equals("org.junit.jupiter.engine.JupiterTestEngine"))
110+
.map(ServiceLoader.Provider::get)
111+
.findFirst()
112+
.orElseThrow(
113+
() ->
114+
new IllegalStateException(
115+
"JUnit Jupiter engine not found; add junit-jupiter-engine to the classpath"));
116+
}
117+
}

‎randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/SysProps.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,14 @@ public enum SysProps {
2121
*/
2222
TESTS_RANDOM_ASSERTING("tests.random.asserting"),
2323

24+
/**
25+
* Test reiteration count for the experimental test engine that re-runs full suites multiple times
26+
* (with a constant or varying seed).
27+
*
28+
* @see RepeatExecutionTestEngine
29+
*/
30+
TESTS_ITERS("tests.iters"),
31+
2432
/**
2533
* A "multiplier" for certain methods that return random values in {@link RandomizedTest}.
2634
*

‎randomizedtesting-jupiter/src/main/java/com/carrotsearch/randomizedtesting/jupiter/internals/RandomizedContextImpl.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.carrotsearch.randomizedtesting.jupiter.FixSeed;
44
import com.carrotsearch.randomizedtesting.jupiter.Hashing;
55
import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
6+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
67
import com.carrotsearch.randomizedtesting.jupiter.Seed;
78
import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
89
import com.carrotsearch.randomizedtesting.jupiter.SysProps;
@@ -15,6 +16,7 @@
1516
import java.util.Random;
1617
import java.util.function.LongFunction;
1718
import org.junit.jupiter.api.extension.ExtensionContext;
19+
import org.junit.platform.engine.UniqueId;
1820

1921
public final class RandomizedContextImpl implements Closeable, RandomizedContext {
2022
private final RandomizedContextImpl parent;
@@ -106,7 +108,7 @@ RandomizedContextImpl deriveNew(ExtensionContext extensionContext) {
106108
throw new RuntimeException(
107109
String.format(
108110
Locale.ROOT,
109-
"@%s annotatoin must declare concrete seeds or seed chains on: %s",
111+
"@%s annotation must declare concrete seeds or seed chains on: %s",
110112
FixSeed.class.getName(),
111113
extensionContext.getElement().get()));
112114
}
@@ -119,7 +121,19 @@ RandomizedContextImpl deriveNew(ExtensionContext extensionContext) {
119121
var nextSeed = firstAndRest.first();
120122
var remainingChain = firstAndRest.rest();
121123
if (nextSeed.isUnspecified()) {
122-
nextSeed = new Seed(this.seed.value() ^ Hashing.hash(extensionContext.getUniqueId()));
124+
var uniqueId = UniqueId.parse(extensionContext.getUniqueId());
125+
var strippedId = uniqueId.toString();
126+
if (Objects.equals(
127+
RepeatExecutionTestEngine.ENGINE_ID, uniqueId.getEngineId().orElse(null))) {
128+
var segments = uniqueId.getSegments();
129+
segments = segments.subList(2, segments.size());
130+
var stripped = UniqueId.root(segments.getFirst().getType(), segments.getFirst().getValue());
131+
for (int i = 1; i < segments.size(); i++) {
132+
stripped = stripped.append(segments.get(i));
133+
}
134+
strippedId = stripped.toString();
135+
}
136+
nextSeed = new Seed(this.seed.value() ^ Hashing.hash(strippedId));
123137
}
124138

125139
return new RandomizedContextImpl(

‎randomizedtesting-jupiter/src/main/java/module-info.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension;
1+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
22

33
module com.carrotsearch.randomizedtesting {
44
requires org.junit.jupiter.api;
@@ -14,5 +14,13 @@
1414
org.junit.platform.commons;
1515

1616
provides org.junit.jupiter.api.extension.Extension with
17-
RandomizedContextExtension;
17+
com.carrotsearch.randomizedtesting.jupiter.internals.RandomizedContextExtension;
18+
19+
// These entries install support for tests.iters (RepeatedExecutionTestEngine).
20+
requires org.junit.platform.engine;
21+
22+
uses org.junit.platform.engine.TestEngine;
23+
24+
provides org.junit.platform.engine.TestEngine with
25+
RepeatExecutionTestEngine;
1826
}

‎randomizedtesting-jupiter/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.java‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package com.carrotsearch.randomizedtesting.tests;
2+
3+
import static com.carrotsearch.randomizedtesting.tests.infra.TestInfra.*;
4+
import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
5+
import static org.junit.platform.testkit.engine.EventConditions.event;
6+
import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure;
7+
import static org.junit.platform.testkit.engine.EventConditions.test;
8+
9+
import com.carrotsearch.randomizedtesting.jupiter.Randomized;
10+
import com.carrotsearch.randomizedtesting.jupiter.RandomizedContext;
11+
import com.carrotsearch.randomizedtesting.jupiter.RepeatExecutionTestEngine;
12+
import com.carrotsearch.randomizedtesting.jupiter.SeedChain;
13+
import com.carrotsearch.randomizedtesting.jupiter.SysProps;
14+
import com.carrotsearch.randomizedtesting.tests.infra.IgnoreInStandaloneRuns;
15+
import com.carrotsearch.randomizedtesting.tests.infra.TestInfra;
16+
import java.io.PrintWriter;
17+
import java.util.List;
18+
import org.assertj.core.api.Assertions;
19+
import org.junit.jupiter.api.Test;
20+
21+
/** Verifies that {@link RepeatExecutionTestEngine} correctly multiplies test execution. */
22+
public class F007_TestReiteration {
23+
@Test
24+
void noReiterationsByDefault() {
25+
var result =
26+
collectExecutionResults(
27+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
28+
.selectors(selectClass(SimpleTest.class)));
29+
30+
result
31+
.results()
32+
.testEvents()
33+
.assertThatEvents()
34+
.doNotHave(event(finishedWithFailure()))
35+
.haveExactly(0, event(test()));
36+
}
37+
38+
@Test
39+
void testsAreMultipliedByIterationCount() {
40+
var result =
41+
collectExecutionResults(
42+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
43+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "3")
44+
.selectors(selectClass(SimpleTest.class)));
45+
46+
result.results().testEvents().assertStatistics(s -> s.finished(3).succeeded(3));
47+
}
48+
49+
@Test
50+
void seedsAreIdenticalAcrossIterationsWithFixedRootSeed() {
51+
var iterations = 5;
52+
var result =
53+
collectExecutionResults(
54+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
55+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
56+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
57+
.selectors(selectClass(SimpleTest.class)));
58+
59+
result
60+
.results()
61+
.testEvents()
62+
.assertStatistics(s -> s.finished(iterations).succeeded(iterations));
63+
64+
// Each iteration should have the same seed chain because we strip the top-level reiteration
65+
// segments.
66+
Assertions.assertThat(result.capturedOutput().values().stream().distinct())
67+
.as("seed chains should be the same across iterations")
68+
.hasSize(1);
69+
70+
Assertions.assertThat(
71+
result.capturedOutput().values().stream()
72+
.map(value -> value.split("\\s")[0])
73+
.map(v -> SeedChain.parse(v))
74+
.map(chain -> chain.seeds().getFirst().toString()))
75+
.allMatch(v -> v.equals("DEADBEEF"));
76+
}
77+
78+
@Test
79+
void seedsAreDifferentAcrossIterationsWithNoRootSeed() {
80+
var iterations = 5;
81+
var result =
82+
collectExecutionResults(
83+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
84+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "" + iterations)
85+
.selectors(selectClass(SimpleTest.class)));
86+
87+
result
88+
.results()
89+
.testEvents()
90+
.assertStatistics(s -> s.finished(iterations).succeeded(iterations));
91+
92+
// Each iteration should have a random root seed if there is no top-level fixed seed.
93+
Assertions.assertThat(result.capturedOutput().values().stream().distinct())
94+
.as("seed chains should be different across iterations")
95+
.hasSizeBetween(iterations - 2, iterations);
96+
}
97+
98+
@Test
99+
void randomnessIsIdenticalForJupiterAndReiteratedTests() {
100+
List<String> jupiterResults;
101+
List<String> repeatedExecutionResults;
102+
103+
{
104+
var result =
105+
collectExecutionResults(
106+
TestInfra.testKitBuilder()
107+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
108+
.selectors(selectClass(SimpleTest.class)));
109+
110+
result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
111+
112+
jupiterResults = result.capturedOutput().values().stream().toList();
113+
}
114+
115+
{
116+
var result =
117+
collectExecutionResults(
118+
TestInfra.testKitBuilder(RepeatExecutionTestEngine.ENGINE_ID)
119+
.configurationParameter(SysProps.TESTS_ITERS.propertyKey, "1")
120+
.configurationParameter(SysProps.TESTS_SEED.propertyKey, "DEADBEEF")
121+
.selectors(selectClass(SimpleTest.class)));
122+
123+
result.results().testEvents().assertStatistics(s -> s.finished(1).succeeded(1));
124+
125+
repeatedExecutionResults = result.capturedOutput().values().stream().toList();
126+
}
127+
128+
Assertions.assertThat(jupiterResults).containsExactlyElementsOf(repeatedExecutionResults);
129+
}
130+
131+
@Randomized
132+
static class SimpleTest extends IgnoreInStandaloneRuns {
133+
@Test
134+
void test(PrintWriter pw, RandomizedContext ctx) {
135+
pw.println(ctx.getSeedChain() + " " + ctx.getRandom().nextLong());
136+
}
137+
}
138+
}

‎randomizedtesting-jupiter/src/test/java/com/carrotsearch/randomizedtesting/tests/F007_TestReiteration.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Feature: re-run (reiterate) one or more tests multiple times, with constant or varying seeds.
2+
3+
## Functionality
4+
5+
* It should be possible to "rerun" one or more tests multiple times with the same or varying seed to check if a
6+
problematic
7+
failure depends on the seed (reproduces) or if it's not reproducible.
8+
9+
* The reiteration is controlled by a system property `tests.iters`, taking the number of reiterations to execute.
10+
11+
* Only junit jupiter tests are reiterated at the moment (by wrapping jupiter test engine and delegating execution to
12+
it).
13+
14+
* If `tests.seed` is fixed, all reiterations should result in the same randomness and execution results. Otherwise,
15+
each reiteration starts with a random root seed.
16+
17+
## Migration notes (from randomizedtesting for junit4)
18+
19+
* Test reiteration will run the full stack of all extensions and hooks - it's not a mere re-execution of an
20+
individual test.

0 commit comments

Comments
 (0)