Skip to content

Commit 79e85c5

Browse files
baevbaev
authored
support composed flaky, muted, and severity annotations across Java test adapters (via #1331)
1 parent 4940f1d commit 79e85c5

24 files changed

Lines changed: 885 additions & 40 deletions

File tree

‎allure-java-commons/src/main/java/io/qameta/allure/util/AnnotationUtils.java‎

Lines changed: 89 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import io.qameta.allure.LabelAnnotation;
2020
import io.qameta.allure.LinkAnnotation;
2121
import io.qameta.allure.Muted;
22+
import io.qameta.allure.Severity;
23+
import io.qameta.allure.SeverityLevel;
2224
import io.qameta.allure.model.Label;
2325
import io.qameta.allure.model.Link;
2426
import org.slf4j.Logger;
@@ -32,6 +34,7 @@
3234
import java.util.Collection;
3335
import java.util.HashSet;
3436
import java.util.Objects;
37+
import java.util.Optional;
3538
import java.util.Set;
3639
import java.util.function.BiFunction;
3740
import java.util.stream.Collectors;
@@ -46,6 +49,7 @@
4649
* test cases via reflection.
4750
*
4851
*/
52+
@SuppressWarnings("PMD.GodClass")
4953
public final class AnnotationUtils {
5054

5155
private static final Logger LOGGER = LoggerFactory.getLogger(AnnotationUtils.class);
@@ -57,23 +61,77 @@ private AnnotationUtils() {
5761
}
5862

5963
/**
60-
* Returns true if {@link io.qameta.allure.Flaky} annotation is present.
64+
* Returns true if {@link io.qameta.allure.Flaky} annotation is present. The annotation is
65+
* also discovered when used as a meta annotation on other (custom) annotations.
6166
*
6267
* @param annotatedElement the element to search annotations on.
6368
* @return true if {@link io.qameta.allure.Flaky} annotation is present, false otherwise.
6469
*/
6570
public static boolean isFlaky(final AnnotatedElement annotatedElement) {
66-
return annotatedElement.isAnnotationPresent(Flaky.class);
71+
return isFlaky(asList(annotatedElement.getAnnotations()));
6772
}
6873

6974
/**
70-
* Returns true if {@link io.qameta.allure.Muted} annotation is present.
75+
* Returns true if {@link io.qameta.allure.Flaky} annotation is present within given annotations,
76+
* either directly or as a meta annotation.
77+
*
78+
* @param annotations annotations to analyse.
79+
* @return true if {@link io.qameta.allure.Flaky} annotation is present, false otherwise.
80+
*/
81+
public static boolean isFlaky(final Collection<Annotation> annotations) {
82+
return findMetaAnnotations(Flaky.class, annotations).findAny().isPresent();
83+
}
84+
85+
/**
86+
* Returns true if {@link io.qameta.allure.Muted} annotation is present. The annotation is
87+
* also discovered when used as a meta annotation on other (custom) annotations.
7188
*
7289
* @param annotatedElement the element to search annotations on.
7390
* @return true if {@link io.qameta.allure.Muted} annotation is present, false otherwise.
7491
*/
7592
public static boolean isMuted(final AnnotatedElement annotatedElement) {
76-
return annotatedElement.isAnnotationPresent(Muted.class);
93+
return isMuted(asList(annotatedElement.getAnnotations()));
94+
}
95+
96+
/**
97+
* Returns true if {@link io.qameta.allure.Muted} annotation is present within given annotations,
98+
* either directly or as a meta annotation.
99+
*
100+
* @param annotations annotations to analyse.
101+
* @return true if {@link io.qameta.allure.Muted} annotation is present, false otherwise.
102+
*/
103+
public static boolean isMuted(final Collection<Annotation> annotations) {
104+
return findMetaAnnotations(Muted.class, annotations).findAny().isPresent();
105+
}
106+
107+
/**
108+
* Returns the severity specified by the {@link io.qameta.allure.Severity} annotation. The
109+
* annotation is also discovered when used as a meta annotation on other (custom) annotations.
110+
*
111+
* @param annotatedElement the element to search annotations on.
112+
* @return the discovered severity, if any.
113+
*/
114+
public static Optional<SeverityLevel> getSeverity(final AnnotatedElement annotatedElement) {
115+
return getSeverity(asList(annotatedElement.getAnnotations()));
116+
}
117+
118+
/**
119+
* Returns the severity specified by the {@link io.qameta.allure.Severity} annotation within
120+
* given annotations. A directly declared {@link io.qameta.allure.Severity} takes precedence;
121+
* otherwise the annotation is also discovered when used as a meta annotation on other (custom)
122+
* annotations.
123+
*
124+
* @param annotations annotations to analyse.
125+
* @return the discovered severity, if any.
126+
*/
127+
public static Optional<SeverityLevel> getSeverity(final Collection<Annotation> annotations) {
128+
return annotations.stream()
129+
.filter(Severity.class::isInstance)
130+
.map(Severity.class::cast)
131+
.findFirst()
132+
.map(Optional::of)
133+
.orElseGet(() -> findMetaAnnotations(Severity.class, annotations).findFirst())
134+
.map(Severity::value);
77135
}
78136

79137
/**
@@ -169,6 +227,33 @@ private static <T, U extends Annotation> Stream<T> extractMetaAnnotations(
169227
return Stream.empty();
170228
}
171229

230+
// Unlike AnnotatedElement#getAnnotationsByType, discovers annotations of the given type even
231+
// when they are used as a meta annotation on another (custom) annotation.
232+
private static <T extends Annotation> Stream<T> findMetaAnnotations(
233+
final Class<T> annotationType,
234+
final Collection<Annotation> candidates) {
235+
final Set<Annotation> visited = new HashSet<>();
236+
return candidates.stream()
237+
.flatMap(AnnotationUtils::extractRepeatable)
238+
.flatMap(candidate -> findMetaAnnotations(annotationType, candidate, visited));
239+
}
240+
241+
private static <T extends Annotation> Stream<T> findMetaAnnotations(
242+
final Class<T> annotationType,
243+
final Annotation candidate,
244+
final Set<Annotation> visited) {
245+
if (isInJavaLangAnnotationPackage(candidate.annotationType()) || !visited.add(candidate)) {
246+
return Stream.empty();
247+
}
248+
final Stream<T> current = annotationType.equals(candidate.annotationType())
249+
? Stream.of(annotationType.cast(candidate))
250+
: Stream.empty();
251+
final Stream<T> children = Stream.of(candidate.annotationType().getAnnotations())
252+
.flatMap(AnnotationUtils::extractRepeatable)
253+
.flatMap(annotation -> findMetaAnnotations(annotationType, annotation, visited));
254+
return Stream.concat(current, children);
255+
}
256+
172257
private static Stream<Label> extractLabels(final LabelAnnotation m, final Annotation annotation) {
173258
if (Objects.equals(m.value(), LabelAnnotation.DEFAULT_VALUE)) {
174259
return callValueMethod(annotation)

‎allure-java-commons/src/test/java/io/qameta/allure/util/AnnotationUtilsTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@
1919
import io.qameta.allure.Epic;
2020
import io.qameta.allure.Epics;
2121
import io.qameta.allure.Feature;
22+
import io.qameta.allure.Flaky;
2223
import io.qameta.allure.Issue;
2324
import io.qameta.allure.Issues;
2425
import io.qameta.allure.LabelAnnotation;
2526
import io.qameta.allure.Link;
2627
import io.qameta.allure.LinkAnnotation;
2728
import io.qameta.allure.Links;
29+
import io.qameta.allure.Muted;
2830
import io.qameta.allure.Owner;
31+
import io.qameta.allure.Severity;
32+
import io.qameta.allure.SeverityLevel;
2933
import io.qameta.allure.Story;
3034
import io.qameta.allure.TmsLink;
3135
import io.qameta.allure.TmsLinks;
@@ -42,6 +46,9 @@
4246
import static io.qameta.allure.Allure.step;
4347
import static io.qameta.allure.util.AnnotationUtils.getLabels;
4448
import static io.qameta.allure.util.AnnotationUtils.getLinks;
49+
import static io.qameta.allure.util.AnnotationUtils.getSeverity;
50+
import static io.qameta.allure.util.AnnotationUtils.isFlaky;
51+
import static io.qameta.allure.util.AnnotationUtils.isMuted;
4552
import static io.qameta.allure.util.ResultsUtils.EPIC_LABEL_NAME;
4653
import static io.qameta.allure.util.ResultsUtils.FEATURE_LABEL_NAME;
4754
import static io.qameta.allure.util.ResultsUtils.STORY_LABEL_NAME;
@@ -497,6 +504,99 @@ void inheritedLinkAnnotationsTest() {
497504
);
498505
}
499506

507+
@Flaky
508+
@Muted
509+
@Severity(SeverityLevel.MINOR)
510+
static class WithDirectStatusAnnotations {
511+
}
512+
513+
static class WithoutStatusAnnotations {
514+
}
515+
516+
@Test
517+
void shouldDetectDirectFlakyMutedSeverity() {
518+
assertThat(isFlaky(WithDirectStatusAnnotations.class)).isTrue();
519+
assertThat(isMuted(WithDirectStatusAnnotations.class)).isTrue();
520+
assertThat(getSeverity(WithDirectStatusAnnotations.class)).contains(SeverityLevel.MINOR);
521+
}
522+
523+
@Test
524+
void shouldReturnDefaultsWhenStatusAnnotationsAbsent() {
525+
assertThat(isFlaky(WithoutStatusAnnotations.class)).isFalse();
526+
assertThat(isMuted(WithoutStatusAnnotations.class)).isFalse();
527+
assertThat(getSeverity(WithoutStatusAnnotations.class)).isEmpty();
528+
}
529+
530+
@Retention(RetentionPolicy.RUNTIME)
531+
@Target({ElementType.METHOD, ElementType.TYPE})
532+
@Flaky
533+
@Muted
534+
@Severity(SeverityLevel.CRITICAL)
535+
@interface CriticalUnstable {
536+
}
537+
538+
@CriticalUnstable
539+
static class WithComposedStatusAnnotation {
540+
}
541+
542+
@Test
543+
void shouldDetectFlakyMutedSeverityViaMetaAnnotation() {
544+
assertThat(isFlaky(WithComposedStatusAnnotation.class)).isTrue();
545+
assertThat(isMuted(WithComposedStatusAnnotation.class)).isTrue();
546+
assertThat(getSeverity(WithComposedStatusAnnotation.class)).contains(SeverityLevel.CRITICAL);
547+
}
548+
549+
@CriticalUnstable
550+
@Severity(SeverityLevel.MINOR)
551+
static class ComposedBeforeDirectSeverity {
552+
}
553+
554+
@Severity(SeverityLevel.MINOR)
555+
@CriticalUnstable
556+
static class DirectBeforeComposedSeverity {
557+
}
558+
559+
@Test
560+
void directSeverityShouldTakePrecedenceOverMetaAnnotation() {
561+
// regardless of declaration order, the directly declared @Severity(MINOR) must win over
562+
// the CRITICAL severity contributed by the @CriticalUnstable meta annotation
563+
assertThat(getSeverity(ComposedBeforeDirectSeverity.class)).contains(SeverityLevel.MINOR);
564+
assertThat(getSeverity(DirectBeforeComposedSeverity.class)).contains(SeverityLevel.MINOR);
565+
}
566+
567+
@Retention(RetentionPolicy.RUNTIME)
568+
@Target({ElementType.METHOD, ElementType.TYPE})
569+
@CriticalUnstable
570+
@interface NestedUnstable {
571+
}
572+
573+
@NestedUnstable
574+
static class WithNestedComposedStatusAnnotation {
575+
}
576+
577+
@Test
578+
void shouldDetectFlakyMutedSeverityViaNestedMetaAnnotation() {
579+
assertThat(isFlaky(WithNestedComposedStatusAnnotation.class)).isTrue();
580+
assertThat(isMuted(WithNestedComposedStatusAnnotation.class)).isTrue();
581+
assertThat(getSeverity(WithNestedComposedStatusAnnotation.class)).contains(SeverityLevel.CRITICAL);
582+
}
583+
584+
@Flaky
585+
@Muted
586+
@Severity(SeverityLevel.BLOCKER)
587+
static class UnstableParent {
588+
}
589+
590+
static class UnstableChild extends UnstableParent {
591+
}
592+
593+
@Test
594+
void shouldInheritStatusAnnotationsFromSuperclass() {
595+
assertThat(isFlaky(UnstableChild.class)).isTrue();
596+
assertThat(isMuted(UnstableChild.class)).isTrue();
597+
assertThat(getSeverity(UnstableChild.class)).contains(SeverityLevel.BLOCKER);
598+
}
599+
500600
private static Set<Label> getLabelsFor(final Class<?> annotatedClass) {
501601
return step(
502602
"Extract labels from annotated class",

‎allure-junit-platform/src/main/java/io/qameta/allure/junitplatform/AllureJunitPlatform.java‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
import io.qameta.allure.AllureLifecycle;
2121
import io.qameta.allure.AttachmentOptions;
2222
import io.qameta.allure.Description;
23-
import io.qameta.allure.Severity;
24-
import io.qameta.allure.SeverityLevel;
2523
import io.qameta.allure.model.Label;
2624
import io.qameta.allure.model.Parameter;
2725
import io.qameta.allure.model.Status;
@@ -610,12 +608,22 @@ private void startTest(final TestIdentifier testIdentifier,
610608

611609
result.setDescription(description);
612610

613-
testMethod.map(this::getSeverity)
611+
testMethod.map(AnnotationUtils::getSeverity)
614612
.filter(Optional::isPresent)
615-
.orElse(testClass.flatMap(this::getSeverity))
613+
.orElse(testClass.flatMap(AnnotationUtils::getSeverity))
616614
.map(ResultsUtils::createSeverityLabel)
617615
.ifPresent(result.getLabels()::add);
618616

617+
final boolean flaky = testMethod.map(AnnotationUtils::isFlaky).orElse(false)
618+
|| testClass.map(AnnotationUtils::isFlaky).orElse(false);
619+
final boolean muted = testMethod.map(AnnotationUtils::isMuted).orElse(false)
620+
|| testClass.map(AnnotationUtils::isMuted).orElse(false);
621+
result.setStatusDetails(
622+
new StatusDetails()
623+
.setFlaky(flaky)
624+
.setMuted(muted)
625+
);
626+
619627
testMethod.ifPresent(
620628
method -> ResultsUtils.processDescription(
621629
method.getDeclaringClass().getClassLoader(),
@@ -781,12 +789,6 @@ private List<TestIdentifier> getParents(final TestIdentifier testIdentifier) {
781789
return result;
782790
}
783791

784-
private Optional<SeverityLevel> getSeverity(final AnnotatedElement annotatedElement) {
785-
return getAnnotations(annotatedElement, Severity.class)
786-
.map(Severity::value)
787-
.findAny();
788-
}
789-
790792
private Optional<String> getDisplayName(final AnnotatedElement annotatedElement) {
791793
return getAnnotations(annotatedElement, DisplayName.class)
792794
.map(DisplayName::value)

‎allure-junit-platform/src/test/java/io/qameta/allure/junitplatform/AllureJunitPlatformTest.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,10 @@
2727
import io.qameta.allure.junitplatform.features.DisabledTests;
2828
import io.qameta.allure.junitplatform.features.DynamicTests;
2929
import io.qameta.allure.junitplatform.features.FailedTests;
30+
import io.qameta.allure.junitplatform.features.FlakyMutedTest;
3031
import io.qameta.allure.junitplatform.features.JupiterUniqueIdTest;
3132
import io.qameta.allure.junitplatform.features.MarkerAnnotationSupport;
33+
import io.qameta.allure.junitplatform.features.MetaAnnotationTest;
3234
import io.qameta.allure.junitplatform.features.NestedDisplayNameTests;
3335
import io.qameta.allure.junitplatform.features.NestedTests;
3436
import io.qameta.allure.junitplatform.features.OneTest;
@@ -809,6 +811,47 @@ void shouldSetSeverity() {
809811
.contains("trivial");
810812
}
811813

814+
@AllureFeatures.MarkerAnnotations
815+
@Test
816+
void shouldSetFlakyAndMuted() {
817+
final AllureResults results = runClasses(FlakyMutedTest.class);
818+
assertThat(results.getTestResults())
819+
.filteredOn("name", "passedFlakyMuted()")
820+
.extracting(tr -> tr.getStatusDetails().isFlaky(), tr -> tr.getStatusDetails().isMuted())
821+
.containsExactly(tuple(true, true));
822+
}
823+
824+
@AllureFeatures.MarkerAnnotations
825+
@Test
826+
void shouldKeepFlakyAndMutedForFailedTest() {
827+
final AllureResults results = runClasses(FlakyMutedTest.class);
828+
assertThat(results.getTestResults())
829+
.filteredOn("name", "failedFlakyMuted()")
830+
.extracting(
831+
TestResult::getStatus,
832+
tr -> tr.getStatusDetails().isFlaky(),
833+
tr -> tr.getStatusDetails().isMuted()
834+
)
835+
.containsExactly(tuple(Status.FAILED, true, true));
836+
}
837+
838+
@AllureFeatures.MarkerAnnotations
839+
@Test
840+
void shouldSupportFlakyMutedSeverityAsMetaAnnotation() {
841+
final AllureResults results = runClasses(MetaAnnotationTest.class);
842+
final List<TestResult> testResults = results.getTestResults();
843+
844+
assertThat(testResults)
845+
.extracting(tr -> tr.getStatusDetails().isFlaky(), tr -> tr.getStatusDetails().isMuted())
846+
.containsExactly(tuple(true, true));
847+
848+
assertThat(testResults)
849+
.flatExtracting(TestResult::getLabels)
850+
.filteredOn("name", SEVERITY_LABEL_NAME)
851+
.extracting(Label::getValue)
852+
.containsExactly("critical");
853+
}
854+
812855
@AllureFeatures.MarkerAnnotations
813856
@Test
814857
void shouldSetOwner() {

0 commit comments

Comments
 (0)