Skip to content

Commit 493d1e3

Browse files
daniel-mohedanodaniel-mohedano
authored
Avoid retrying untracked Karate scenarios (#12012)
fix: avoid retrying untracked karate scenarios fix: merge test scenarios Co-authored-by: daniel.mohedano <daniel.mohedano@datadoghq.com>
1 parent c7901a5 commit 493d1e3

21 files changed

Lines changed: 1536 additions & 11 deletions

File tree

‎dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateExecutionInstrumentation.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ public void methodAdvice(MethodTransformer transformer) {
7676
public static class RetryAdvice {
7777
@Advice.OnMethodEnter
7878
public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
79+
if (KarateTracingHook.skipTracking(scenarioRuntime)) {
80+
return;
81+
}
82+
7983
ExecutionContext executionContext =
8084
InstrumentationContext.get(Scenario.class, ExecutionContext.class)
8185
.computeIfAbsent(scenarioRuntime.scenario, ExecutionContext::create);
@@ -91,6 +95,10 @@ public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
9195

9296
@Advice.OnMethodExit
9397
public static void afterExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
98+
if (KarateTracingHook.skipTracking(scenarioRuntime)) {
99+
return;
100+
}
101+
94102
if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) {
95103
// nested call
96104
return;

‎dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ private static boolean skipTracking(FeatureRuntime fr) {
233233
return !fr.caller.isNone();
234234
}
235235

236-
private static boolean skipTracking(ScenarioRuntime sr) {
236+
public static boolean skipTracking(ScenarioRuntime sr) {
237237
// do not track nested scenario calls and setup scenarios
238238
return !sr.caller.isNone() || sr.tags.getTagKeys().contains("setup");
239239
}

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ class KarateTest extends CiVisibilityInstrumentationTest {
8585
}
8686

8787
def "test early flakiness detection #testcaseName"() {
88+
Assumptions.assumeTrue(assumption)
89+
8890
givenEarlyFlakinessDetectionEnabled(true)
8991
givenKnownTests(knownTestsList)
9092

@@ -93,16 +95,18 @@ class KarateTest extends CiVisibilityInstrumentationTest {
9395
assertSpansData(testcaseName)
9496

9597
where:
96-
testcaseName | tests | knownTestsList
97-
"test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestFQN("[org/example/test_succeed_one_case] test succeed", "first scenario")]
98+
testcaseName | tests | knownTestsList | assumption
99+
"test-efd-known-test" | [TestSucceedOneCaseKarate] | [new TestFQN("[org/example/test_succeed_one_case] test succeed", "first scenario")] | true
98100
"test-efd-known-parameterized-test" | [TestParameterizedKarate] | [
99101
new TestFQN("[org/example/test_parameterized] test parameterized", "first scenario as an outline")
100-
]
101-
"test-efd-new-test" | [TestSucceedOneCaseKarate] | []
102-
"test-efd-new-parameterized-test" | [TestParameterizedKarate] | []
103-
"test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] // is executed only twice
104-
"test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | []
105-
"test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | []
102+
] | true
103+
"test-efd-new-test" | [TestSucceedOneCaseKarate] | [] | true
104+
"test-efd-new-parameterized-test" | [TestParameterizedKarate] | [] | true
105+
"test-efd-new-slow-test" | [TestSucceedKarateSlow] | [] | true // is executed only twice
106+
"test-efd-faulty-session-threshold" | [TestParameterizedMoreCasesKarate] | [] | true
107+
"test-efd-skip-new-test" | [TestSucceedKarateSkipEfd] | [] | true
108+
"test-efd-setup" | [TestWithSetupKarate] | [] | KarateUtils.isSetupTagSupported(KarateUtils.getKarateVersion())
109+
"test-efd-called-feature" | [TestSucceedCalledFeatureKarate] | [new TestFQN("[org/example/test_called_feature] test called feature", "caller")] | true
106110
}
107111

108112
def "test quarantined #testcaseName"() {

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestSucceedCalledFeatureKarate.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package org.example;
2+
3+
import com.intuit.karate.junit5.Karate;
4+
5+
public class TestSucceedCalledFeatureKarate {
6+
7+
@Karate.Test
8+
public Karate testCalledFeature() {
9+
return Karate.run("classpath:org/example/test_called_feature.feature");
10+
}
11+
}

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: test called feature
2+
3+
Scenario: caller
4+
* call read('test_called_feature_nested.feature')

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_called_feature_nested.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: test called feature nested
2+
3+
Scenario: called
4+
* def value = true

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-called-feature/events.ftl‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"component" : "karate",
7+
"env" : "none",
8+
"library_version" : ${content_meta_library_version},
9+
"step.name" : "* call read('test_called_feature_nested.feature')"
10+
},
11+
"metrics" : {
12+
"step.endLine" : 4,
13+
"step.startLine" : 4
14+
},
15+
"name" : "karate.step",
16+
"parent_id" : ${content_parent_id},
17+
"resource" : "* call read('test_called_feature_nested.feature')",
18+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
19+
"span_id" : ${content_span_id},
20+
"start" : ${content_start},
21+
"trace_id" : ${content_trace_id}
22+
},
23+
"type" : "span",
24+
"version" : 1
25+
}, {
26+
"content" : {
27+
"duration" : ${content_duration_2},
28+
"error" : 0,
29+
"meta" : {
30+
"component" : "karate",
31+
"env" : "none",
32+
"library_version" : ${content_meta_library_version},
33+
"step.name" : "* def value = true"
34+
},
35+
"metrics" : {
36+
"step.endLine" : 4,
37+
"step.startLine" : 4
38+
},
39+
"name" : "karate.step",
40+
"parent_id" : ${content_span_id},
41+
"resource" : "* def value = true",
42+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
43+
"span_id" : ${content_span_id_2},
44+
"start" : ${content_start_2},
45+
"trace_id" : ${content_trace_id}
46+
},
47+
"type" : "span",
48+
"version" : 1
49+
}, {
50+
"content" : {
51+
"duration" : ${content_duration_3},
52+
"error" : 0,
53+
"meta" : {
54+
"_dd.p.tid" : ${content_meta__dd_p_tid},
55+
"component" : "karate",
56+
"dummy_ci_tag" : "dummy_ci_tag_value",
57+
"env" : "none",
58+
"library_version" : ${content_meta_library_version},
59+
"span.kind" : "test_suite_end",
60+
"test.framework" : "karate",
61+
"test.framework_version" : ${content_meta_test_framework_version},
62+
"test.module" : "karate-1.0",
63+
"test.status" : "pass",
64+
"test.suite" : "[org/example/test_called_feature] test called feature",
65+
"test.type" : "test",
66+
"test_session.name" : "session-name"
67+
},
68+
"metrics" : {
69+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
70+
},
71+
"name" : "karate.test_suite",
72+
"resource" : "[org/example/test_called_feature] test called feature",
73+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
74+
"start" : ${content_start_3},
75+
"test_module_id" : ${content_test_module_id},
76+
"test_session_id" : ${content_test_session_id},
77+
"test_suite_id" : ${content_test_suite_id}
78+
},
79+
"type" : "test_suite_end",
80+
"version" : 1
81+
}, {
82+
"content" : {
83+
"duration" : ${content_duration_4},
84+
"error" : 0,
85+
"meta" : {
86+
"_dd.profiling.ctx" : "test",
87+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
88+
"component" : "karate",
89+
"dummy_ci_tag" : "dummy_ci_tag_value",
90+
"env" : "none",
91+
"language" : "jvm",
92+
"library_version" : ${content_meta_library_version},
93+
"runtime-id" : ${content_meta_runtime_id},
94+
"span.kind" : "test",
95+
"test.final_status" : "pass",
96+
"test.framework" : "karate",
97+
"test.framework_version" : ${content_meta_test_framework_version},
98+
"test.module" : "karate-1.0",
99+
"test.name" : "caller",
100+
"test.status" : "pass",
101+
"test.suite" : "[org/example/test_called_feature] test called feature",
102+
"test.type" : "test",
103+
"test_session.name" : "session-name"
104+
},
105+
"metrics" : {
106+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
107+
"_dd.profiling.enabled" : 0,
108+
"_dd.trace_span_attribute_schema" : 0,
109+
"process_id" : ${content_metrics_process_id}
110+
},
111+
"name" : "karate.test",
112+
"parent_id" : ${content_parent_id_2},
113+
"resource" : "[org/example/test_called_feature] test called feature.caller",
114+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
115+
"span_id" : ${content_parent_id},
116+
"start" : ${content_start_4},
117+
"test_module_id" : ${content_test_module_id},
118+
"test_session_id" : ${content_test_session_id},
119+
"test_suite_id" : ${content_test_suite_id},
120+
"trace_id" : ${content_trace_id}
121+
},
122+
"type" : "test",
123+
"version" : 2
124+
}, {
125+
"content" : {
126+
"duration" : ${content_duration_5},
127+
"error" : 0,
128+
"meta" : {
129+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
130+
"_dd.profiling.ctx" : "test",
131+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
132+
"component" : "karate",
133+
"dummy_ci_tag" : "dummy_ci_tag_value",
134+
"env" : "none",
135+
"language" : "jvm",
136+
"library_version" : ${content_meta_library_version},
137+
"runtime-id" : ${content_meta_runtime_id},
138+
"span.kind" : "test_session_end",
139+
"test.command" : "karate-1.0",
140+
"test.early_flake.enabled" : "true",
141+
"test.framework" : "karate",
142+
"test.framework_version" : ${content_meta_test_framework_version},
143+
"test.status" : "pass",
144+
"test.type" : "test",
145+
"test_session.name" : "session-name"
146+
},
147+
"metrics" : {
148+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
149+
"_dd.profiling.enabled" : 0,
150+
"_dd.trace_span_attribute_schema" : 0,
151+
"process_id" : ${content_metrics_process_id}
152+
},
153+
"name" : "karate.test_session",
154+
"resource" : "karate-1.0",
155+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
156+
"start" : ${content_start_5},
157+
"test_session_id" : ${content_test_session_id}
158+
},
159+
"type" : "test_session_end",
160+
"version" : 1
161+
}, {
162+
"content" : {
163+
"duration" : ${content_duration_6},
164+
"error" : 0,
165+
"meta" : {
166+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
167+
"component" : "karate",
168+
"dummy_ci_tag" : "dummy_ci_tag_value",
169+
"env" : "none",
170+
"library_version" : ${content_meta_library_version},
171+
"span.kind" : "test_module_end",
172+
"test.early_flake.enabled" : "true",
173+
"test.framework" : "karate",
174+
"test.framework_version" : ${content_meta_test_framework_version},
175+
"test.module" : "karate-1.0",
176+
"test.status" : "pass",
177+
"test.type" : "test",
178+
"test_session.name" : "session-name"
179+
},
180+
"metrics" : {
181+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
182+
},
183+
"name" : "karate.test_module",
184+
"resource" : "karate-1.0",
185+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
186+
"start" : ${content_start_6},
187+
"test_module_id" : ${content_test_module_id},
188+
"test_session_id" : ${content_test_session_id}
189+
},
190+
"type" : "test_module_end",
191+
"version" : 1
192+
} ]

‎dd-java-agent/instrumentation/karate/karate-1.0/src/test/resources/test-efd-setup/coverages.ftl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ ]

0 commit comments

Comments
 (0)