Skip to content

Commit 77f9ba5

Browse files
daniel-mohedanoPerfectSlayer
daniel-mohedano
and
PerfectSlayer
authored
Stop retrying aborted Karate scenarios (#12015)
fix: avoid retrying untracked karate scenarios fix: finish karate tests after scenario post-processing fix: stop retrying aborted karate scenarios fix: merge test scenarios Merge branch 'daniel.mohedano/karate-ignore-untracked-scenarios' into daniel.mohedano/karate-finish-after-post-processing Merge branch 'daniel.mohedano/karate-finish-after-post-processing' into daniel.mohedano/karate-stop-aborted-retries fix: honor expected karate failures Merge branch 'daniel.mohedano/karate-finish-after-post-processing' into daniel.mohedano/karate-stop-aborted-retries nit: comment Merge branch 'master' into daniel.mohedano/karate-stop-aborted-retries # Conflicts: # dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java Co-authored-by: bruce.bujon <bruce.bujon@datadoghq.com>
1 parent 9a8af9d commit 77f9ba5

8 files changed

Lines changed: 242 additions & 2 deletions

File tree

‎dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/ExecutionContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ public class ExecutionContext {
1111
private final TestExecutionPolicy executionPolicy;
1212
private boolean suppressFailures;
1313
private Throwable suppressedError;
14+
private boolean testStarted;
1415

1516
public ExecutionContext(TestExecutionPolicy executionPolicy) {
1617
this.executionPolicy = executionPolicy;
@@ -28,6 +29,14 @@ public TestExecutionPolicy getExecutionPolicy() {
2829
return executionPolicy;
2930
}
3031

32+
public void setTestStarted(boolean testStarted) {
33+
this.testStarted = testStarted;
34+
}
35+
36+
public boolean isTestStarted() {
37+
return testStarted;
38+
}
39+
3140
/**
3241
* Karate v2 {@code StepResult} is immutable (no {@code setFailedReason}/{@code setErrorIgnored}),
3342
* so a failure that was suppressed for retry purposes cannot be carried on the replacement step.

‎dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
2424
ExecutionContext executionContext =
2525
InstrumentationContext.get(Scenario.class, ExecutionContext.class)
2626
.computeIfAbsent(scenarioRuntime.getScenario(), ExecutionContext::create);
27+
executionContext.setTestStarted(false);
2728

2829
// Indicate beforehand whether failures should be suppressed. This aligns the ordering with
2930
// the rest of the frameworks.
@@ -42,7 +43,9 @@ public static void afterExecute(
4243
Scenario scenario = scenarioRuntime.getScenario();
4344
ExecutionContext context =
4445
InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
45-
if (context == null) {
46+
if (context == null || !context.isTestStarted()) {
47+
// avoid retrying an aborted scenario that did not start, as the execution policy will not
48+
// advance and loop
4649
return;
4750
}
4851
KarateTracingListener.afterScenario(scenarioRuntime, result, context);
@@ -58,7 +61,11 @@ public static void afterExecute(
5861
while (executionPolicy.applicable()) {
5962
ScenarioRuntime retry =
6063
new ScenarioRuntime(scenarioRuntime.getFeatureRuntime(), scenario);
61-
finalResult = retry.call();
64+
ScenarioResult retryResult = retry.call();
65+
if (!context.isTestStarted()) {
66+
break;
67+
}
68+
finalResult = retryResult;
6269
}
6370

6471
// override the return value so the final attempt is the one recorded.

‎dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,9 @@ private boolean beforeScenario(ScenarioRunEvent event) {
155155
TestSourceData.UNKNOWN,
156156
null,
157157
executionTracker);
158+
if (context != null) {
159+
context.setTestStarted(true);
160+
}
158161
return true;
159162
}
160163

‎dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ class KarateV2Test extends CiVisibilityInstrumentationTest {
6868
testcaseName | success | tests | retriedTests
6969
"test-failed" | false | [TestFailedKarate] | []
7070
"test-retry-failed" | false | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")]
71+
"test-retry-abort-suite" | false | [TestFailedAbortSuiteKarate] | [new TestFQN("[org/example/test_abort_suite] test abort suite", "aborting scenario")]
7172
"test-failed-then-succeed" | true | [TestFailedThenSucceedKarate] | [new TestFQN("[org/example/test_failed_then_succeed] test failed", "flaky scenario")]
7273
"test-retry-after-scenario-failed" | false | [TestFailedAfterScenarioKarate] | [
7374
new TestFQN("[org/example/test_after_scenario_failed] test after scenario failed", "after scenario failed")

‎dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestFailedAbortSuiteKarate.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package org.example;
2+
3+
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
5+
import com.intuit.karate.Results;
6+
import com.intuit.karate.Runner;
7+
import org.junit.jupiter.api.Test;
8+
9+
public class TestFailedAbortSuiteKarate {
10+
11+
@Test
12+
public void test() {
13+
Results results = Runner.path("classpath:org/example/test_abort_suite.feature").parallel(1);
14+
assertEquals(0, results.getFailCount(), results.getErrorMessages());
15+
}
16+
}

‎dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_abort_suite.feature‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Feature: test abort suite
2+
3+
@fail
4+
Scenario: aborting scenario
5+
* configure abortSuiteOnFailure = true
6+
* match 1 == 1

‎dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-retry-abort-suite/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

‎dd-java-agent/instrumentation/karate/karate-2.0/src/test/resources/test-retry-abort-suite/events.ftl‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"component" : "karate",
7+
"env" : "none",
8+
"library_version" : ${content_meta_library_version},
9+
"step.name" : "* 1 == 1"
10+
},
11+
"metrics" : {
12+
"step.endLine" : 6,
13+
"step.startLine" : 6
14+
},
15+
"name" : "karate.step",
16+
"parent_id" : ${content_parent_id},
17+
"resource" : "* 1 == 1",
18+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
19+
"span_id" : ${content_span_id},
20+
"start" : ${content_start},
21+
"trace_id" : ${content_trace_id}
22+
},
23+
"type" : "span",
24+
"version" : 1
25+
}, {
26+
"content" : {
27+
"duration" : ${content_duration_2},
28+
"error" : 0,
29+
"meta" : {
30+
"component" : "karate",
31+
"env" : "none",
32+
"library_version" : ${content_meta_library_version},
33+
"step.name" : "* abortSuiteOnFailure = true"
34+
},
35+
"metrics" : {
36+
"step.endLine" : 5,
37+
"step.startLine" : 5
38+
},
39+
"name" : "karate.step",
40+
"parent_id" : ${content_parent_id},
41+
"resource" : "* abortSuiteOnFailure = true",
42+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
43+
"span_id" : ${content_span_id_2},
44+
"start" : ${content_start_2},
45+
"trace_id" : ${content_trace_id}
46+
},
47+
"type" : "span",
48+
"version" : 1
49+
}, {
50+
"content" : {
51+
"duration" : ${content_duration_3},
52+
"error" : 1,
53+
"meta" : {
54+
"_dd.p.tid" : ${content_meta__dd_p_tid},
55+
"component" : "karate",
56+
"dummy_ci_tag" : "dummy_ci_tag_value",
57+
"env" : "none",
58+
"error.message" : ${content_meta_error_message},
59+
"error.stack" : ${content_meta_error_stack},
60+
"error.type" : "java.lang.RuntimeException",
61+
"library_version" : ${content_meta_library_version},
62+
"span.kind" : "test_suite_end",
63+
"test.framework" : "karate",
64+
"test.framework_version" : ${content_meta_test_framework_version},
65+
"test.module" : "karate-2.0",
66+
"test.status" : "fail",
67+
"test.suite" : "[org/example/test_abort_suite] test abort suite",
68+
"test.type" : "test",
69+
"test_session.name" : "session-name"
70+
},
71+
"metrics" : {
72+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
73+
},
74+
"name" : "karate.test_suite",
75+
"resource" : "[org/example/test_abort_suite] test abort suite",
76+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
77+
"start" : ${content_start_3},
78+
"test_module_id" : ${content_test_module_id},
79+
"test_session_id" : ${content_test_session_id},
80+
"test_suite_id" : ${content_test_suite_id}
81+
},
82+
"type" : "test_suite_end",
83+
"version" : 1
84+
}, {
85+
"content" : {
86+
"duration" : ${content_duration_4},
87+
"error" : 1,
88+
"meta" : {
89+
"_dd.profiling.ctx" : "test",
90+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
91+
"component" : "karate",
92+
"dummy_ci_tag" : "dummy_ci_tag_value",
93+
"env" : "none",
94+
"error.message" : ${content_meta_error_message},
95+
"error.stack" : ${content_meta_error_stack_2},
96+
"error.type" : "java.lang.RuntimeException",
97+
"language" : "jvm",
98+
"library_version" : ${content_meta_library_version},
99+
"runtime-id" : ${content_meta_runtime_id},
100+
"span.kind" : "test",
101+
"test.failure_suppressed" : "true",
102+
"test.framework" : "karate",
103+
"test.framework_version" : ${content_meta_test_framework_version},
104+
"test.module" : "karate-2.0",
105+
"test.name" : "aborting scenario",
106+
"test.status" : "fail",
107+
"test.suite" : "[org/example/test_abort_suite] test abort suite",
108+
"test.traits" : "{\"category\":[\"fail\"]}",
109+
"test.type" : "test",
110+
"test_session.name" : "session-name"
111+
},
112+
"metrics" : {
113+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
114+
"_dd.profiling.enabled" : 0,
115+
"_dd.trace_span_attribute_schema" : 0,
116+
"process_id" : ${content_metrics_process_id}
117+
},
118+
"name" : "karate.test",
119+
"parent_id" : ${content_parent_id_2},
120+
"resource" : "[org/example/test_abort_suite] test abort suite.aborting scenario",
121+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
122+
"span_id" : ${content_parent_id},
123+
"start" : ${content_start_4},
124+
"test_module_id" : ${content_test_module_id},
125+
"test_session_id" : ${content_test_session_id},
126+
"test_suite_id" : ${content_test_suite_id},
127+
"trace_id" : ${content_trace_id}
128+
},
129+
"type" : "test",
130+
"version" : 2
131+
}, {
132+
"content" : {
133+
"duration" : ${content_duration_5},
134+
"error" : 0,
135+
"meta" : {
136+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
137+
"_dd.profiling.ctx" : "test",
138+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
139+
"component" : "karate",
140+
"dummy_ci_tag" : "dummy_ci_tag_value",
141+
"env" : "none",
142+
"language" : "jvm",
143+
"library_version" : ${content_meta_library_version},
144+
"runtime-id" : ${content_meta_runtime_id},
145+
"span.kind" : "test_session_end",
146+
"test.command" : "karate-2.0",
147+
"test.framework" : "karate",
148+
"test.framework_version" : ${content_meta_test_framework_version},
149+
"test.status" : "fail",
150+
"test.type" : "test",
151+
"test_session.name" : "session-name"
152+
},
153+
"metrics" : {
154+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
155+
"_dd.profiling.enabled" : 0,
156+
"_dd.trace_span_attribute_schema" : 0,
157+
"process_id" : ${content_metrics_process_id}
158+
},
159+
"name" : "karate.test_session",
160+
"resource" : "karate-2.0",
161+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
162+
"start" : ${content_start_5},
163+
"test_session_id" : ${content_test_session_id}
164+
},
165+
"type" : "test_session_end",
166+
"version" : 1
167+
}, {
168+
"content" : {
169+
"duration" : ${content_duration_6},
170+
"error" : 0,
171+
"meta" : {
172+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
173+
"component" : "karate",
174+
"dummy_ci_tag" : "dummy_ci_tag_value",
175+
"env" : "none",
176+
"library_version" : ${content_meta_library_version},
177+
"span.kind" : "test_module_end",
178+
"test.framework" : "karate",
179+
"test.framework_version" : ${content_meta_test_framework_version},
180+
"test.module" : "karate-2.0",
181+
"test.status" : "fail",
182+
"test.type" : "test",
183+
"test_session.name" : "session-name"
184+
},
185+
"metrics" : {
186+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
187+
},
188+
"name" : "karate.test_module",
189+
"resource" : "karate-2.0",
190+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
191+
"start" : ${content_start_6},
192+
"test_module_id" : ${content_test_module_id},
193+
"test_session_id" : ${content_test_session_id}
194+
},
195+
"type" : "test_module_end",
196+
"version" : 1
197+
} ]

0 commit comments

Comments
 (0)