Skip to content

Commit 479bf20

Browse files
MilanTyagi2004MilanTyagi2004
authored
Fix empty and missing testcase entries for JUnit 5 @BeforeAll/@afterall failures (#3329)
* Fix: JUnit 5 @BeforeAll and @afterall failures produce empty testcase name
1 parent 0c380a2 commit 479bf20

10 files changed

Lines changed: 180 additions & 26 deletions

File tree

‎maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,9 +276,11 @@ private void mergeTestHistoryResult() {
276276
List<TestMethodStats> testMethodStats = entry.getValue();
277277
String testClassMethodName = entry.getKey();
278278

279-
// Handle @BeforeAll failures (null, empty, or ends with ".null" method names)
279+
// Handle @BeforeAll failures (null, empty, ends with ".null" or ".initializationError" method names)
280280
// But only if they actually failed (ERROR or FAILURE), not if they were skipped
281-
if ((StringUtils.isBlank(testClassMethodName) || testClassMethodName.endsWith(".null"))
281+
if ((StringUtils.isBlank(testClassMethodName)
282+
|| testClassMethodName.endsWith(".null")
283+
|| testClassMethodName.endsWith(".initializationError"))
282284
&& (testClassMethodName == null || !testClassMethodName.contains("$"))) {
283285

284286
// Check if this is actually a failure/error (not skipped or success)

‎maven-surefire-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -406,9 +406,9 @@ private TestResultType getTestResultTypeWithBeforeAllHandling(
406406
Map<String, List<WrappedReportEntry>> methodStatistics) {
407407
TestResultType resultType = getTestResultType(methodRuns);
408408

409-
// Special handling for @BeforeAll failures (null method name or method name is "null")
409+
// Special handling for @BeforeAll failures (null method name or method name is "null" or "initializationError")
410410
// If @BeforeAll failed but any actual test methods succeeded, treat it as a flake
411-
if ((methodName == null || methodName.equals("null"))
411+
if ((methodName == null || methodName.equals("null") || methodName.equals("initializationError"))
412412
&& (resultType == TestResultType.ERROR || resultType == TestResultType.FAILURE)) {
413413
// Check if any actual test methods (non-null and not "null" names) succeeded
414414
boolean hasSuccessfulTestMethods = methodStatistics.entrySet().stream()

‎surefire-its/src/test/java/org/apache/maven/surefire/its/JUnit5BeforeAllContainerFailureIT.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.maven.surefire.its;
20+
21+
import org.apache.maven.surefire.its.fixture.OutputValidator;
22+
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
23+
import org.junit.jupiter.api.Test;
24+
25+
/**
26+
* Integration test for JUnit 5 container-level failures (e.g. @BeforeAll).
27+
* Verifies that such failures generate a valid valid testcase named
28+
* "initializationError" in the XML report.
29+
*/
30+
public class JUnit5BeforeAllContainerFailureIT extends SurefireJUnit4IntegrationTestCase {
31+
32+
@Test
33+
public void testBeforeAllContainerFailure() {
34+
OutputValidator outputValidator = unpack("junit5-beforeall-container-failure")
35+
.setJUnitVersion("5.9.1")
36+
.maven()
37+
.debugLogging()
38+
.withFailure()
39+
.executeTest();
40+
41+
// One test fails at the container level: meaning tests run = 1, errors = 1, failures = 0, skipped = 0
42+
outputValidator.assertTestSuiteResults(1, 1, 0, 0);
43+
44+
outputValidator
45+
.getSurefireReportsXmlFile("TEST-junitplatform.AlwaysFailingBeforeAllTest.xml")
46+
.assertContainsText("tests=\"1\" errors=\"1\" skipped=\"0\" failures=\"0\"")
47+
.assertContainsText(
48+
"<testcase name=\"initializationError\" classname=\"junitplatform.AlwaysFailingBeforeAllTest\" time=")
49+
.assertContainsText("<error message=\"BeforeAll always fails\" type=\"java.lang.RuntimeException\">")
50+
.assertContainsText("<![CDATA[java.lang.RuntimeException: BeforeAll always fails")
51+
.assertContainsText(
52+
"at junitplatform.AlwaysFailingBeforeAllTest.setup(AlwaysFailingBeforeAllTest.java:10)");
53+
}
54+
}

‎surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire943ReportContentIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ private void validateFailInBeforeClass(OutputValidator validator, String classNa
6969
Xpp3Dom child = children[0];
7070

7171
Assertions.assertEquals(className, child.getAttribute("classname"));
72-
Assertions.assertEquals("", child.getAttribute("name"));
72+
Assertions.assertEquals("initializationError", child.getAttribute("name"));
7373

7474
Assertions.assertEquals(
7575
1,

‎surefire-its/src/test/resources/junit-platform-rerun-failing-before-all/src/test/java/junitplatform/AlwaysFailingTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,4 @@ static void setup() {
1919
public void testThatNeverRuns() {
2020
System.out.println("This test never runs because beforeAll always fails");
2121
}
22-
}
22+
}

‎surefire-its/src/test/resources/junit5-beforeall-container-failure/pom.xml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns="http://maven.apache.org/POM/4.0.0"
22+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<groupId>org.apache.maven.plugins.surefire</groupId>
27+
<artifactId>junit5-beforeall-container-failure</artifactId>
28+
<version>1.0-SNAPSHOT</version>
29+
<name>Test for BeforeAll container failures in JUnit 5/Platform</name>
30+
31+
<properties>
32+
<maven.compiler.source>1.8</maven.compiler.source>
33+
<maven.compiler.target>1.8</maven.compiler.target>
34+
</properties>
35+
36+
<dependencies>
37+
<dependency>
38+
<groupId>org.junit.jupiter</groupId>
39+
<artifactId>junit-jupiter-api</artifactId>
40+
<version>${junit.version}</version>
41+
<scope>test</scope>
42+
</dependency>
43+
<dependency>
44+
<groupId>org.junit.jupiter</groupId>
45+
<artifactId>junit-jupiter-engine</artifactId>
46+
<version>${junit.version}</version>
47+
<scope>test</scope>
48+
</dependency>
49+
</dependencies>
50+
51+
<build>
52+
<pluginManagement>
53+
<plugins>
54+
<plugin>
55+
<groupId>org.apache.maven.plugins</groupId>
56+
<artifactId>maven-surefire-plugin</artifactId>
57+
<version>${surefire.version}</version>
58+
</plugin>
59+
</plugins>
60+
</pluginManagement>
61+
<plugins>
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-surefire-plugin</artifactId>
65+
</plugin>
66+
</plugins>
67+
</build>
68+
</project>

‎surefire-its/src/test/resources/junit5-beforeall-container-failure/src/test/java/junitplatform/AlwaysFailingBeforeAllTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package junitplatform;
2+
3+
import org.junit.jupiter.api.BeforeAll;
4+
import org.junit.jupiter.api.Test;
5+
6+
public class AlwaysFailingBeforeAllTest {
7+
8+
@BeforeAll
9+
public static void setup() {
10+
throw new RuntimeException("BeforeAll always fails");
11+
}
12+
13+
@Test
14+
public void testPassingTest() {
15+
// this test should be skipped or fail due to beforeAll failure
16+
}
17+
}

‎surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@
5858
* @since 2.22.0
5959
*/
6060
final class RunListenerAdapter implements TestExecutionListener, TestOutputReceiver<OutputReportEntry>, RunModeSetter {
61+
private static final String INITIALIZATION_ERROR = "initializationError";
62+
6163
private final ClassMethodIndexer classMethodIndexer = new ClassMethodIndexer();
6264
private final ConcurrentMap<TestIdentifier, Long> testStartTime = new ConcurrentHashMap<>();
6365
private final ConcurrentMap<TestIdentifier, TestExecutionResult> failures = new ConcurrentHashMap<>();
@@ -236,6 +238,15 @@ private SimpleReportEntry createReportEntry(
236238
boolean failed = testExecutionResult == null || testExecutionResult.getStatus() == FAILED;
237239
String methodName = failed || testIdentifier.isTest() ? classMethodName.getMethodSignature() : null;
238240
String methodText = failed || testIdentifier.isTest() ? classMethodName.getMethodDisplayName() : null;
241+
242+
if (testExecutionResult != null
243+
&& testExecutionResult.getStatus() != org.junit.platform.engine.TestExecutionResult.Status.SUCCESSFUL
244+
&& testIdentifier.isContainer()
245+
&& methodName == null) {
246+
methodName = INITIALIZATION_ERROR;
247+
methodText = INITIALIZATION_ERROR;
248+
}
249+
239250
if (Objects.equals(methodName, methodText)) {
240251
methodText = null;
241252
}
@@ -304,7 +315,8 @@ private TestIdentifier findTopParent(TestIdentifier testIdentifier) {
304315
}
305316

306317
/**
307-
* Checks if the test identifier has a parent ID but using reflection as it's only available from 1.8.
318+
* Checks if the test identifier has a parent ID but using reflection as it's
319+
* only available from 1.8.
308320
*
309321
* @param testIdentifier the test identifier to check
310322
* @return true if the test identifier has a parent ID, false otherwise
@@ -342,18 +354,19 @@ private TestIdentifier findFirstParentContainerAndSourceClass(TestIdentifier tes
342354

343355
/**
344356
* <ul>
345-
* <li>[0] class name - used in stacktrace parser</li>
346-
* <li>[1] class display name</li>
347-
* <li>[2] method signature - used in stacktrace parser</li>
348-
* <li>[3] method display name</li>
357+
* <li>[0] class name - used in stacktrace parser</li>
358+
* <li>[1] class display name</li>
359+
* <li>[2] method signature - used in stacktrace parser</li>
360+
* <li>[3] method display name</li>
349361
* </ul>
350362
*
351363
* @param testIdentifier a class or method
352364
* @return 4 elements string array
353365
*/
354366
private ResultDisplay toClassMethodName(TestIdentifier testIdentifier) {
355367

356-
// find the first class or method source in the hierarchy just below the root level
368+
// find the first class or method source in the hierarchy just below the root
369+
// level
357370
// without parent and with ClassSource
358371
Optional<String> classLevelName = Optional.empty();
359372
TestIdentifier parent = findTopParent(testIdentifier);
@@ -413,18 +426,18 @@ private ResultDisplay toClassMethodName(TestIdentifier testIdentifier) {
413426
String methodDisp = hasDisplayName ? methodDisplay : methodDesc;
414427

415428
// The behavior of methods getLegacyReportingName() and getDisplayName().
416-
// junit4 || legacy | display
429+
// junit4 || legacy | display
417430
// ==============||==========|==========
418-
// normal || m | m
419-
// param || m[0] | m[0]
420-
// param+displ || m[displ] | m[displ]
431+
// normal || m | m
432+
// param || m[0] | m[0]
433+
// param+displ || m[displ] | m[displ]
421434

422-
// junit5 || legacy | display
435+
// junit5 || legacy | display
423436
// ==============||==========|==========
424-
// normal || m() | m()
425-
// normal+displ || m() | displ
426-
// param || m()[1] | [1] <param>
427-
// param+displ || m()[1] | displ
437+
// normal || m() | m()
438+
// normal+displ || m() | displ
439+
// param || m()[1] | [1] <param>
440+
// param+displ || m()[1] | displ
428441

429442
return new ResultDisplay(
430443
classLevelName.orElse(source.getClassName()),

‎surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public void shouldFailClassOnBeforeAll() throws Exception {
130130

131131
assertThat(testCaptor.getValue().getSourceName()).isEqualTo(FailingBeforeAllJupiterTest.class.getName());
132132

133-
assertThat(testCaptor.getValue().getName()).isNull();
133+
assertThat(testCaptor.getValue().getName()).isEqualTo("initializationError");
134134

135135
assertThat(testSetCaptor.getAllValues().get(1).getSourceName())
136136
.isEqualTo(FailingBeforeAllJupiterTest.class.getName());
@@ -171,7 +171,7 @@ public void shouldErrorClassOnBeforeAll() throws Exception {
171171
assertThat(testCaptor.getValue().getSourceName())
172172
.isEqualTo(FailingWithErrorBeforeAllJupiterTest.class.getName());
173173

174-
assertThat(testCaptor.getValue().getName()).isNull();
174+
assertThat(testCaptor.getValue().getName()).isEqualTo("initializationError");
175175

176176
assertThat(testSetCaptor.getAllValues().get(1).getSourceName())
177177
.isEqualTo(FailingWithErrorBeforeAllJupiterTest.class.getName());

‎surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/RunListenerAdapterTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ public void notifiedWhenClassExecutionAborted() {
394394
TestSkippedException t = new TestSkippedException("skipped");
395395
adapter.executionFinished(newClassIdentifier(), aborted(t));
396396
String source = MyTestClass.class.getName();
397-
StackTraceWriter stw = new DefaultStackTraceWriter(source, null, t);
397+
StackTraceWriter stw = new DefaultStackTraceWriter(source, "initializationError", t);
398398
ArgumentCaptor<SimpleReportEntry> report = ArgumentCaptor.forClass(SimpleReportEntry.class);
399399
verify(listener).testSetCompleted(report.capture());
400400
assertThat(report.getValue().getSourceName()).isEqualTo(source);
@@ -438,7 +438,7 @@ public void notifiedWithCorrectNamesWhenClassExecutionFailed() {
438438

439439
ReportEntry entry = entryCaptor.getValue();
440440
assertEquals(MyTestClass.class.getTypeName(), entry.getSourceName());
441-
assertNull(entry.getName());
441+
assertEquals("initializationError", entry.getName());
442442
assertNotNull(entry.getStackTraceWriter());
443443
assertNotNull(entry.getStackTraceWriter().getThrowable());
444444
assertThat(entry.getStackTraceWriter().getThrowable().getTarget()).isInstanceOf(AssertionError.class);
@@ -457,7 +457,7 @@ public void notifiedWithCorrectNamesWhenClassExecutionErrored() {
457457

458458
ReportEntry entry = entryCaptor.getValue();
459459
assertEquals(MyTestClass.class.getTypeName(), entry.getSourceName());
460-
assertNull(entry.getName());
460+
assertEquals("initializationError", entry.getName());
461461
assertNotNull(entry.getStackTraceWriter());
462462
assertNotNull(entry.getStackTraceWriter().getThrowable());
463463
assertThat(entry.getStackTraceWriter().getThrowable().getTarget()).isInstanceOf(RuntimeException.class);

0 commit comments

Comments
 (0)