Finish Karate tests after scenario post-processing (#12014)

fix: avoid retrying untracked karate scenarios

fix: finish karate tests after scenario post-processing

fix: merge test scenarios

Merge branch 'daniel.mohedano/karate-ignore-untracked-scenarios' into daniel.mohedano/karate-finish-after-post-processing

fix: honor expected karate failures

Merge branch 'master' into daniel.mohedano/karate-finish-after-post-processing

# Conflicts:
#	dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java

Co-authored-by: daniel.mohedano <daniel.mohedano@datadoghq.com>

Author: daniel-mohedano

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateExecutionInstrumentation.java

@@ -11,7 +11,6 @@
 import com.google.auto.service.AutoService;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
-import datadog.trace.api.Config;
 import java.util.Collections;
 import java.util.Map;
 
@@ -37,11 +36,6 @@ public KarateExecutionInstrumentation() {
     super("ci-visibility", "karate", "test-retry");
   }
 
-  @Override
-  public boolean isEnabled() {
-    return super.isEnabled() && Config.get().isCiVisibilityExecutionPoliciesEnabled();
-  }
-
   @Override
   public String[] knownMatchingTypes() {
     return new String[] {"io.karatelabs.core.ScenarioRuntime", "io.karatelabs.core.ScenarioResult"};

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java

@@ -39,19 +39,20 @@ public static void afterExecute(
         return;
       }
 
+      Scenario scenario = scenarioRuntime.getScenario();
+      ExecutionContext context =
+          InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
+      if (context == null) {
+        return;
+      }
+      KarateTracingListener.afterScenario(scenarioRuntime, result, context);
+
       if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) {
-        // nested call (a retry invoked below, or a called scenario)
+        // retry invoked below
         return;
       }
 
       try {
-        Scenario scenario = scenarioRuntime.getScenario();
-        ExecutionContext context =
-            InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
-        if (context == null) {
-          return;
-        }
-
         ScenarioResult finalResult = result;
         TestExecutionPolicy executionPolicy = context.getExecutionPolicy();
         while (executionPolicy.applicable()) {
@@ -79,7 +80,8 @@ public static void onAddingStepResult(
         @Advice.Argument(value = 0, readOnly = false) StepResult stepResult,
         @Advice.FieldValue("scenario") Scenario scenario) {
 
-      if (stepResult.isFailed()) {
+      // Keep expected failures intact so Karate can apply the @fail result inversion.
+      if (stepResult.isFailed() && !scenario.isFail()) {
         ExecutionContext executionContext =
             InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
         if (executionContext == null) {

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java

@@ -61,6 +61,15 @@ public boolean onEvent(RunEvent event) {
     return true;
   }
 
+  private void afterScenario(ScenarioRunEvent event) {
+    ScenarioRuntime sr = event.source();
+    Scenario scenario = sr.getScenario();
+    // A context means call advice will finish the test after scenario post-processing.
+    if (scenarioContext.get(scenario) == null) {
+      afterScenario(sr, event.result(), null);
+    }
+  }
+
   private boolean beforeFeature(FeatureRunEvent event) {
     FeatureRuntime fr = event.source();
     if (skipTracking(fr)) {
@@ -149,20 +158,22 @@ private boolean beforeScenario(ScenarioRunEvent event) {
     return true;
   }
 
-  private void afterScenario(ScenarioRunEvent event) {
-    ScenarioRuntime sr = event.source();
+  public static void afterScenario(
+      ScenarioRuntime sr, ScenarioResult result, ExecutionContext context) {
     if (skipTracking(sr)) {
       return;
     }
     Scenario scenario = sr.getScenario();
-    ScenarioResult result = event.result();
     TestDescriptor testDescriptor = KarateUtils.toTestDescriptor(sr);
 
-    ExecutionContext context = scenarioContext.get(scenario);
     Throwable suppressedError = context != null ? context.getAndClearSuppressedError() : null;
-    Throwable failedReason = getFailedReason(result, suppressedError);
+    // @fail retains the expected step error even when the final result passes, so only inspect
+    // failed results for an error
+    boolean resultFailed = result != null && result.isFailed();
+    Throwable failedReason =
+        resultFailed ? getFailedReason(result, suppressedError) : suppressedError;
 
-    if ((result != null && result.isFailed()) || failedReason != null) {
+    if (resultFailed || failedReason != null) {
       TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, failedReason);
     } else if (result == null || result.getStepResults().isEmpty()) {
       TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, null);

dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2RetryDisabledForkedTest.groovy

@@ -0,0 +1,42 @@
+import datadog.trace.api.DisableTestTrace
+import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.instrumentation.karate2.KarateUtils
+import datadog.trace.instrumentation.karate2.TestEventsHandlerHolder
+import org.example.TestSucceedKarate
+
+@DisableTestTrace(reason = "avoid self-tracing")
+class KarateV2RetryDisabledForkedTest extends CiVisibilityInstrumentationTest {
+
+  @Override
+  void configurePreAgent() {
+    super.configurePreAgent()
+    injectSysConfig("trace.test-retry.enabled", "false")
+  }
+
+  def "finishes scenarios without execution policies when retry instrumentation is disabled"() {
+    setup:
+    givenEarlyFlakinessDetectionEnabled(true)
+    givenKnownTests([])
+    TestEventsHandlerHolder.start()
+
+    when:
+    new TestSucceedKarate().test()
+    assertSpansData("test-retry-disabled")
+
+    then:
+    noExceptionThrown()
+
+    cleanup:
+    TestEventsHandlerHolder.stop()
+  }
+
+  @Override
+  String instrumentedLibraryName() {
+    return "karate"
+  }
+
+  @Override
+  String instrumentedLibraryVersion() {
+    return KarateUtils.getKarateVersion()
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy

@@ -33,6 +33,8 @@ class KarateV2Test extends CiVisibilityInstrumentationTest {
     "test-with-setup"       | true    | [TestWithSetupKarate]
     "test-parameterized"    | true    | [TestParameterizedKarate]
     "test-failed"           | false   | [TestFailedKarate]
+    "test-fail-expected"    | true    | [TestFailedExpectedKarate]
+    "test-fail-unexpected"  | false   | [TestFailedUnexpectedKarate]
     "test-skipped-feature"  | true    | [TestSkippedFeatureKarate]
   }
 
@@ -67,6 +69,10 @@ class KarateV2Test extends CiVisibilityInstrumentationTest {
     "test-failed"              | false   | [TestFailedKarate]              | []
     "test-retry-failed"        | false   | [TestFailedKarate]              | [new TestFQN("[org/example/test_failed] test failed", "second scenario")]
     "test-failed-then-succeed" | true    | [TestFailedThenSucceedKarate]   | [new TestFQN("[org/example/test_failed_then_succeed] test failed", "flaky scenario")]
+    "test-retry-after-scenario-failed" | false | [TestFailedAfterScenarioKarate] | [
+      new TestFQN("[org/example/test_after_scenario_failed] test after scenario failed", "after scenario failed")
+    ]
+    "test-retry-fail-expected" | true | [TestFailedExpectedKarate] | [new TestFQN("[org/example/test_fail_expected] test fail expected", "expected failure")]
     "test-retry-parameterized" | false   | [TestFailedParameterizedKarate] | [
       new TestFQN("[org/example/test_failed_parameterized] test parameterized", "first scenario as an outline")
     ]

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestFailedAfterScenarioKarate.java

@@ -0,0 +1,17 @@
+package org.example;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.intuit.karate.Results;
+import com.intuit.karate.Runner;
+import org.junit.jupiter.api.Test;
+
+public class TestFailedAfterScenarioKarate {
+
+  @Test
+  public void test() {
+    Results results =
+        Runner.path("classpath:org/example/test_after_scenario_failed.feature").parallel(1);
+    assertEquals(0, results.getFailCount(), results.getErrorMessages());
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestFailedExpectedKarate.java

@@ -0,0 +1,16 @@
+package org.example;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.intuit.karate.Results;
+import com.intuit.karate.Runner;
+import org.junit.jupiter.api.Test;
+
+public class TestFailedExpectedKarate {
+
+  @Test
+  public void test() {
+    Results results = Runner.path("classpath:org/example/test_fail_expected.feature").parallel(1);
+    assertEquals(0, results.getFailCount(), results.getErrorMessages());
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestFailedUnexpectedKarate.java

@@ -0,0 +1,16 @@
+package org.example;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import com.intuit.karate.Results;
+import com.intuit.karate.Runner;
+import org.junit.jupiter.api.Test;
+
+public class TestFailedUnexpectedKarate {
+
+  @Test
+  public void test() {
+    Results results = Runner.path("classpath:org/example/test_fail_unexpected.feature").parallel(1);
+    assertEquals(0, results.getFailCount(), results.getErrorMessages());
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_after_scenario_failed.feature

@@ -0,0 +1,5 @@
+Feature: test after scenario failed
+
+  Scenario: after scenario failed
+    * configure afterScenario = function() { karate.fail('after scenario failed') }
+    * def value = true

dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_fail_expected.feature

@@ -0,0 +1,5 @@
+Feature: test fail expected
+
+  @fail
+  Scenario: expected failure
+    * match 1 == 2