Skip to content

Commit 986199e

Browse files
daniel-mohedanodaniel-mohedano
committed
fix: fail testng session on new flaky (efd) independent of result order
1 parent 5ab378f commit 986199e

10 files changed

Lines changed: 475 additions & 0 deletions

File tree

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/execution/EarlyFlakeDetection.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ public boolean suppressFailures() {
7171
return suppressFailures;
7272
}
7373

74+
@Override
75+
public boolean propagateFailure() {
76+
// used to bypass TestNG's RetryAnalyzer marking `fail` + `pass` test executions as passed
77+
return !suppressFailures && results == ExecutionAggregation.MIXED;
78+
}
79+
7480
private int getExecutions(long durationMillis) {
7581
for (ExecutionsByDuration e : executionsByDuration) {
7682
if (durationMillis <= e.getDurationMillis()) {

‎dd-java-agent/instrumentation/testng/testng-7.0/src/test/resources/test-efd-new-flaky-test-fail-pass/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

‎dd-java-agent/instrumentation/testng/testng-7.0/src/test/resources/test-efd-new-flaky-test-fail-pass/events.ftl‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"component" : "testng",
8+
"dummy_ci_tag" : "dummy_ci_tag_value",
9+
"env" : "none",
10+
"library_version" : ${content_meta_library_version},
11+
"span.kind" : "test_suite_end",
12+
"test.codeowners" : "[\"owner1\",\"owner2\"]",
13+
"test.framework" : "testng",
14+
"test.framework_version" : ${content_meta_test_framework_version},
15+
"test.module" : "testng-7.0",
16+
"test.source.file" : "dummy_source_path",
17+
"test.status" : "fail",
18+
"test.suite" : "org.example.TestFailAndThenSucceed",
19+
"test.type" : "test",
20+
"test_session.name" : "session-name"
21+
},
22+
"metrics" : {
23+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count},
24+
"test.source.end" : 19,
25+
"test.source.start" : 11
26+
},
27+
"name" : "testng.test_suite",
28+
"resource" : "org.example.TestFailAndThenSucceed",
29+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
30+
"start" : ${content_start},
31+
"test_module_id" : ${content_test_module_id},
32+
"test_session_id" : ${content_test_session_id},
33+
"test_suite_id" : ${content_test_suite_id}
34+
},
35+
"type" : "test_suite_end",
36+
"version" : 1
37+
}, {
38+
"content" : {
39+
"duration" : ${content_duration_2},
40+
"error" : 1,
41+
"meta" : {
42+
"_dd.profiling.ctx" : "test",
43+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
44+
"component" : "testng",
45+
"dummy_ci_tag" : "dummy_ci_tag_value",
46+
"env" : "none",
47+
"error.message" : ${content_meta_error_message},
48+
"error.stack" : ${content_meta_error_stack},
49+
"error.type" : "java.lang.AssertionError",
50+
"language" : "jvm",
51+
"library_version" : ${content_meta_library_version},
52+
"runtime-id" : ${content_meta_runtime_id},
53+
"span.kind" : "test",
54+
"test.codeowners" : "[\"owner1\",\"owner2\"]",
55+
"test.framework" : "testng",
56+
"test.framework_version" : ${content_meta_test_framework_version},
57+
"test.is_new" : "true",
58+
"test.module" : "testng-7.0",
59+
"test.name" : "test_fail_and_then_succeed",
60+
"test.source.file" : "dummy_source_path",
61+
"test.source.method" : "test_fail_and_then_succeed()V",
62+
"test.status" : "fail",
63+
"test.suite" : "org.example.TestFailAndThenSucceed",
64+
"test.type" : "test",
65+
"test_session.name" : "session-name"
66+
},
67+
"metrics" : {
68+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
69+
"_dd.profiling.enabled" : 0,
70+
"_dd.trace_span_attribute_schema" : 0,
71+
"process_id" : ${content_metrics_process_id},
72+
"test.source.end" : 18,
73+
"test.source.start" : 12
74+
},
75+
"name" : "testng.test",
76+
"parent_id" : ${content_parent_id},
77+
"resource" : "org.example.TestFailAndThenSucceed.test_fail_and_then_succeed",
78+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
79+
"span_id" : ${content_span_id},
80+
"start" : ${content_start_2},
81+
"test_module_id" : ${content_test_module_id},
82+
"test_session_id" : ${content_test_session_id},
83+
"test_suite_id" : ${content_test_suite_id},
84+
"trace_id" : ${content_trace_id}
85+
},
86+
"type" : "test",
87+
"version" : 2
88+
}, {
89+
"content" : {
90+
"duration" : ${content_duration_3},
91+
"error" : 0,
92+
"meta" : {
93+
"_dd.profiling.ctx" : "test",
94+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
95+
"component" : "testng",
96+
"dummy_ci_tag" : "dummy_ci_tag_value",
97+
"env" : "none",
98+
"language" : "jvm",
99+
"library_version" : ${content_meta_library_version},
100+
"runtime-id" : ${content_meta_runtime_id},
101+
"span.kind" : "test",
102+
"test.codeowners" : "[\"owner1\",\"owner2\"]",
103+
"test.final_status" : "fail",
104+
"test.framework" : "testng",
105+
"test.framework_version" : ${content_meta_test_framework_version},
106+
"test.is_new" : "true",
107+
"test.is_retry" : "true",
108+
"test.module" : "testng-7.0",
109+
"test.name" : "test_fail_and_then_succeed",
110+
"test.retry_reason" : "early_flake_detection",
111+
"test.source.file" : "dummy_source_path",
112+
"test.source.method" : "test_fail_and_then_succeed()V",
113+
"test.status" : "pass",
114+
"test.suite" : "org.example.TestFailAndThenSucceed",
115+
"test.type" : "test",
116+
"test_session.name" : "session-name"
117+
},
118+
"metrics" : {
119+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
120+
"_dd.profiling.enabled" : 0,
121+
"_dd.trace_span_attribute_schema" : 0,
122+
"process_id" : ${content_metrics_process_id},
123+
"test.source.end" : 18,
124+
"test.source.start" : 12
125+
},
126+
"name" : "testng.test",
127+
"parent_id" : ${content_parent_id},
128+
"resource" : "org.example.TestFailAndThenSucceed.test_fail_and_then_succeed",
129+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
130+
"span_id" : ${content_span_id_2},
131+
"start" : ${content_start_3},
132+
"test_module_id" : ${content_test_module_id},
133+
"test_session_id" : ${content_test_session_id},
134+
"test_suite_id" : ${content_test_suite_id},
135+
"trace_id" : ${content_trace_id_2}
136+
},
137+
"type" : "test",
138+
"version" : 2
139+
}, {
140+
"content" : {
141+
"duration" : ${content_duration_4},
142+
"error" : 0,
143+
"meta" : {
144+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
145+
"_dd.profiling.ctx" : "test",
146+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
147+
"component" : "testng",
148+
"dummy_ci_tag" : "dummy_ci_tag_value",
149+
"env" : "none",
150+
"language" : "jvm",
151+
"library_version" : ${content_meta_library_version},
152+
"runtime-id" : ${content_meta_runtime_id},
153+
"span.kind" : "test_session_end",
154+
"test.command" : "testng-7.0",
155+
"test.early_flake.enabled" : "true",
156+
"test.framework" : "testng",
157+
"test.framework_version" : ${content_meta_test_framework_version},
158+
"test.status" : "fail",
159+
"test.type" : "test",
160+
"test_session.name" : "session-name"
161+
},
162+
"metrics" : {
163+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4},
164+
"_dd.profiling.enabled" : 0,
165+
"_dd.trace_span_attribute_schema" : 0,
166+
"process_id" : ${content_metrics_process_id}
167+
},
168+
"name" : "testng.test_session",
169+
"resource" : "testng-7.0",
170+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
171+
"start" : ${content_start_4},
172+
"test_session_id" : ${content_test_session_id}
173+
},
174+
"type" : "test_session_end",
175+
"version" : 1
176+
}, {
177+
"content" : {
178+
"duration" : ${content_duration_5},
179+
"error" : 0,
180+
"meta" : {
181+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
182+
"component" : "testng",
183+
"dummy_ci_tag" : "dummy_ci_tag_value",
184+
"env" : "none",
185+
"library_version" : ${content_meta_library_version},
186+
"span.kind" : "test_module_end",
187+
"test.early_flake.enabled" : "true",
188+
"test.framework" : "testng",
189+
"test.framework_version" : ${content_meta_test_framework_version},
190+
"test.module" : "testng-7.0",
191+
"test.status" : "fail",
192+
"test.type" : "test",
193+
"test_session.name" : "session-name"
194+
},
195+
"metrics" : {
196+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}
197+
},
198+
"name" : "testng.test_module",
199+
"resource" : "testng-7.0",
200+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
201+
"start" : ${content_start_5},
202+
"test_module_id" : ${content_test_module_id},
203+
"test_session_id" : ${content_test_session_id}
204+
},
205+
"type" : "test_module_end",
206+
"version" : 1
207+
} ]

‎dd-java-agent/instrumentation/testng/testng-7.0/src/test/resources/test-efd-new-flaky-test-pass-fail/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)