Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import static datadog.trace.civisibility.Constants.CI_VISIBILITY_INSTRUMENTATION_NAME;

import datadog.trace.api.Config;
import datadog.trace.api.DDTags;
import datadog.trace.api.DDTraceId;
import datadog.trace.api.IdGenerationStrategy;
import datadog.trace.api.civisibility.CIConstants;
Expand All @@ -13,8 +14,10 @@
import datadog.trace.api.civisibility.telemetry.TagValue;
import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
import datadog.trace.api.civisibility.telemetry.tag.EventType;
import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
import datadog.trace.api.civisibility.telemetry.tag.FailedTestReplayEnabled;
import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
Expand All @@ -26,13 +29,15 @@
import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
import datadog.trace.bootstrap.instrumentation.api.TagContext;
import datadog.trace.bootstrap.instrumentation.api.Tags;
import datadog.trace.civisibility.Constants;
import datadog.trace.civisibility.codeowners.Codeowners;
import datadog.trace.civisibility.decorator.TestDecorator;
import datadog.trace.civisibility.source.LinesResolver;
import datadog.trace.civisibility.source.SourcePathResolver;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import javax.annotation.Nullable;

public abstract class AbstractTestSession {
Expand Down Expand Up @@ -188,6 +193,13 @@ private TagValue[] telemetryTags() {
}

protected Collection<TagValue> additionalTelemetryTags() {
return Collections.emptyList();
Set<TagValue> tags = new HashSet<>();
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
tags.add(EarlyFlakeDetectionAbortReason.FAULTY);
}
if (span.getTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY) != null) {
tags.add(FailedTestReplayEnabled.SessionMetric.TRUE);
}
return tags;
Comment on lines -191 to +203
Copy link
Copy Markdown
Contributor Author

@daniel-mohedano daniel-mohedano Sep 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like we always want to follow the same logic in HeadlessTestSession and BuildSystemSessionImpl. Was this previously implemented in overriden methods to avoid performing the logic in ManualApiTestSession?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I cannot recall now, but it's fine to have this in logic in the parent class

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,11 @@ public void end(@Nullable Long endTime) {
}
}

boolean debugInfoCaptured = span.getTag(Tags.ERROR_DEBUG_INFO_CAPTURED) != null;
if (debugInfoCaptured) {
executionResults.setHasFailedTestReplayTests();
}

AgentTracer.closeActive();

onSpanFinish.accept(span);
Expand All @@ -310,9 +315,7 @@ public void end(@Nullable Long endTime) {
span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
retryReason instanceof TagValue ? (TagValue) retryReason : null,
span.getTag(Tags.ERROR_DEBUG_INFO_CAPTURED) != null
? FailedTestReplayEnabled.TestMetric.TRUE
: null,
debugInfoCaptured ? FailedTestReplayEnabled.TestMetric.TRUE : null,
span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))
? BrowserDriver.SELENIUM
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,10 @@ private SignalResponse onModuleExecutionResultReceived(ModuleExecutionResult res
setTag(Tags.TEST_TEST_MANAGEMENT_ENABLED, true);
}

if (result.hasFailedTestReplayTests()) {
setTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY, true);
}

testsSkipped.add(result.getTestsSkippedTotal());

tagsPropagator.mergeTestFrameworks(result.getTestFrameworks());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,9 @@
import datadog.trace.api.civisibility.domain.BuildSessionSettings;
import datadog.trace.api.civisibility.domain.JavaAgent;
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
import datadog.trace.api.civisibility.telemetry.TagValue;
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
import datadog.trace.api.civisibility.telemetry.tag.Provider;
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
import datadog.trace.bootstrap.instrumentation.api.Tags;
import datadog.trace.civisibility.Constants;
import datadog.trace.civisibility.codeowners.Codeowners;
import datadog.trace.civisibility.config.ExecutionSettings;
import datadog.trace.civisibility.config.ExecutionSettingsFactory;
Expand Down Expand Up @@ -196,22 +193,15 @@ private void onModuleFinish(AgentSpan moduleSpan) {
TagMergeSpec.of(Tags.TEST_ITR_TESTS_SKIPPING_TYPE),
TagMergeSpec.of(Tags.TEST_ITR_TESTS_SKIPPING_COUNT, Long::sum),
TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED, Boolean::logicalOr),
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED, Boolean::logicalOr));
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED, Boolean::logicalOr),
TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY, Boolean::logicalOr));
}

@Override
public BuildSessionSettings getSettings() {
return settings;
}

@Override
protected Collection<TagValue> additionalTelemetryTags() {
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
return Collections.singleton(EarlyFlakeDetectionAbortReason.FAULTY);
}
return Collections.emptySet();
}

@Override
public void end(@Nullable Long endTime) {
signalServer.stop();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ private void sendModuleExecutionResult() {
earlyFlakeDetectionEnabled && executionStrategy.isEFDLimitReached();
TestManagementSettings testManagementSettings = executionSettings.getTestManagementSettings();
boolean testManagementEnabled = testManagementSettings.isEnabled();
boolean hasFailedTestReplayTests = executionResults.hasFailedTestReplayTests();
long testsSkippedTotal = executionResults.getTestsSkippedByItr();

signalClient.send(
Expand All @@ -177,6 +178,7 @@ private void sendModuleExecutionResult() {
earlyFlakeDetectionEnabled,
earlyFlakeDetectionFaulty,
testManagementEnabled,
hasFailedTestReplayTests,
testsSkippedTotal,
new TreeSet<>(testFrameworks)));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@ public void end(@Nullable Long endTime) {
setTag(Tags.TEST_TEST_MANAGEMENT_ENABLED, true);
}

if (executionResults.hasFailedTestReplayTests()) {
setTag(DDTags.TEST_HAS_FAILED_TEST_REPLAY, true);
}

super.end(endTime);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,9 @@
import datadog.trace.api.civisibility.config.LibraryCapability;
import datadog.trace.api.civisibility.coverage.CoverageStore;
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
import datadog.trace.api.civisibility.telemetry.TagValue;
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionAbortReason;
import datadog.trace.api.civisibility.telemetry.tag.Provider;
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
import datadog.trace.bootstrap.instrumentation.api.Tags;
import datadog.trace.civisibility.Constants;
import datadog.trace.civisibility.codeowners.Codeowners;
import datadog.trace.civisibility.decorator.TestDecorator;
import datadog.trace.civisibility.domain.AbstractTestSession;
Expand All @@ -22,7 +19,6 @@
import datadog.trace.civisibility.source.SourcePathResolver;
import datadog.trace.civisibility.test.ExecutionStrategy;
import java.util.Collection;
import java.util.Collections;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

Expand Down Expand Up @@ -97,15 +93,8 @@ private void onModuleFinish(AgentSpan moduleSpan) {
TagMergeSpec.of(Tags.TEST_EARLY_FLAKE_ENABLED),
TagMergeSpec.of(Tags.TEST_EARLY_FLAKE_ABORT_REASON),
TagMergeSpec.of(DDTags.CI_ITR_TESTS_SKIPPED),
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED));
}

@Override
protected Collection<TagValue> additionalTelemetryTags() {
if (Constants.EFD_ABORT_REASON_FAULTY.equals(span.getTag(Tags.TEST_EARLY_FLAKE_ABORT_REASON))) {
return Collections.singleton(EarlyFlakeDetectionAbortReason.FAULTY);
}
return Collections.emptySet();
TagMergeSpec.of(Tags.TEST_TEST_MANAGEMENT_ENABLED),
TagMergeSpec.of(DDTags.TEST_HAS_FAILED_TEST_REPLAY));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ public class ModuleExecutionResult extends ModuleSignal {
private static final int EARLY_FLAKE_DETECTION_ENABLED_FLAG = 4;
private static final int EARLY_FLAKE_DETECTION_FAULTY_FLAG = 8;
private static final int TEST_MANAGEMENT_ENABLED_FLAG = 16;
private static final int HAS_FAILED_TEST_REPLAY_TESTS_FLAG = 32;

private final boolean coverageEnabled;
private final boolean testSkippingEnabled;
private final boolean earlyFlakeDetectionEnabled;
private final boolean earlyFlakeDetectionFaulty;
private final boolean testManagementEnabled;
private final boolean hasFailedTestReplayTests;
private final long testsSkippedTotal;
private final Collection<TestFramework> testFrameworks;

Expand All @@ -30,6 +32,7 @@ public ModuleExecutionResult(
boolean earlyFlakeDetectionEnabled,
boolean earlyFlakeDetectionFaulty,
boolean testManagementEnabled,
boolean hasFailedTestReplayTests,
long testsSkippedTotal,
Collection<TestFramework> testFrameworks) {
super(sessionId, moduleId);
Expand All @@ -38,6 +41,7 @@ public ModuleExecutionResult(
this.earlyFlakeDetectionEnabled = earlyFlakeDetectionEnabled;
this.earlyFlakeDetectionFaulty = earlyFlakeDetectionFaulty;
this.testManagementEnabled = testManagementEnabled;
this.hasFailedTestReplayTests = hasFailedTestReplayTests;
this.testsSkippedTotal = testsSkippedTotal;
this.testFrameworks = testFrameworks;
}
Expand All @@ -62,6 +66,10 @@ public boolean isTestManagementEnabled() {
return testManagementEnabled;
}

public boolean hasFailedTestReplayTests() {
return hasFailedTestReplayTests;
}

public long getTestsSkippedTotal() {
return testsSkippedTotal;
}
Expand All @@ -84,6 +92,7 @@ public boolean equals(Object o) {
&& moduleId == that.moduleId
&& coverageEnabled == that.coverageEnabled
&& testSkippingEnabled == that.testSkippingEnabled
&& hasFailedTestReplayTests == that.hasFailedTestReplayTests
&& testsSkippedTotal == that.testsSkippedTotal
&& Objects.equals(testFrameworks, that.testFrameworks);
}
Expand All @@ -95,6 +104,7 @@ public int hashCode() {
moduleId,
coverageEnabled,
testSkippingEnabled,
hasFailedTestReplayTests,
testsSkippedTotal,
testFrameworks);
}
Expand All @@ -108,6 +118,8 @@ public String toString() {
+ moduleId
+ ", coverageEnabled="
+ coverageEnabled
+ ", hasFailedTestReplayTests="
+ hasFailedTestReplayTests
+ ", testSkippingEnabled="
+ testSkippingEnabled
+ ", itrTestsSkipped="
Expand Down Expand Up @@ -142,6 +154,9 @@ public ByteBuffer serialize() {
if (testManagementEnabled) {
flags |= TEST_MANAGEMENT_ENABLED_FLAG;
}
if (hasFailedTestReplayTests) {
flags |= HAS_FAILED_TEST_REPLAY_TESTS_FLAG;
}
s.write(flags);

s.write(testsSkippedTotal);
Expand All @@ -160,6 +175,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) {
boolean earlyFlakeDetectionEnabled = (flags & EARLY_FLAKE_DETECTION_ENABLED_FLAG) != 0;
boolean earlyFlakeDetectionFaulty = (flags & EARLY_FLAKE_DETECTION_FAULTY_FLAG) != 0;
boolean testManagementEnabled = (flags & TEST_MANAGEMENT_ENABLED_FLAG) != 0;
boolean hasFailedTestReplayTests = (flags & HAS_FAILED_TEST_REPLAY_TESTS_FLAG) != 0;

long testsSkippedTotal = Serializer.readLong(buffer);
Collection<TestFramework> testFrameworks =
Expand All @@ -173,6 +189,7 @@ public static ModuleExecutionResult deserialize(ByteBuffer buffer) {
earlyFlakeDetectionEnabled,
earlyFlakeDetectionFaulty,
testManagementEnabled,
hasFailedTestReplayTests,
testsSkippedTotal,
testFrameworks);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package datadog.trace.civisibility.test;

import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.LongAdder;

public class ExecutionResults {

private final LongAdder testsSkippedByItr = new LongAdder();
private final AtomicBoolean hasFailedTestReplayTests = new AtomicBoolean();

public void incrementTestsSkippedByItr() {
testsSkippedByItr.increment();
Expand All @@ -13,4 +15,12 @@ public void incrementTestsSkippedByItr() {
public long getTestsSkippedByItr() {
return testsSkippedByItr.sum();
}

public void setHasFailedTestReplayTests() {
this.hasFailedTestReplayTests.set(true);
}

public boolean hasFailedTestReplayTests() {
return hasFailedTestReplayTests.get();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ class ModuleExecutionResultTest extends Specification {

where:
signal << [
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, false, false, false, false, 0, Collections.emptyList()),
new ModuleExecutionResult(DDTraceId.from(12345), 67890, true, false, true, true, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2"))),
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, true, true, false, false, 2, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework("junit", "5.9.2"))),
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, false, false, false, true, true, 3, Arrays.asList(new TestFramework("junit", null), new TestFramework("junit", "5.9.2"))),
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, true, true, true, true, true, Integer.MAX_VALUE, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework(null, "5.9.2")))
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, false, false, false, false, false, 0, Collections.emptyList()),
new ModuleExecutionResult(DDTraceId.from(12345), 67890, true, false, true, true, true, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2"))),
new ModuleExecutionResult(DDTraceId.from(12345), 67890, false, true, true, false, false, true, 2, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework("junit", "5.9.2"))),
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, false, false, false, true, true, false, 3, Arrays.asList(new TestFramework("junit", null), new TestFramework("junit", "5.9.2"))),
new ModuleExecutionResult(DD128bTraceId.from(12345, 67890), 67890, true, true, true, true, true, true, Integer.MAX_VALUE, Arrays.asList(new TestFramework("junit", "4.13.2"), new TestFramework(null, "5.9.2")))
]
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class SignalServerTest extends Specification {
def "test message send and receive"() {
given:
def signalProcessed = new AtomicBoolean(false)
def signal = new ModuleExecutionResult(DDTraceId.from(123), 456, true, true, false, false, false, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signal = new ModuleExecutionResult(DDTraceId.from(123), 456, true, true, false, false, false, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def server = new SignalServer()
def received = new ArrayList()

Expand Down Expand Up @@ -41,8 +41,8 @@ class SignalServerTest extends Specification {

def "test multiple messages send and receive"() {
given:
def signalA = new ModuleExecutionResult(DDTraceId.from(123), 456, false, false, false, false, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalB = new ModuleExecutionResult(DDTraceId.from(234), 567, true, true, false, false, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalA = new ModuleExecutionResult(DDTraceId.from(123), 456, false, false, false, false, false, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalB = new ModuleExecutionResult(DDTraceId.from(234), 567, true, true, false, false, true, false, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def server = new SignalServer()
def received = new ArrayList()

Expand Down Expand Up @@ -70,8 +70,8 @@ class SignalServerTest extends Specification {

def "test multiple clients send and receive"() {
given:
def signalA = new ModuleExecutionResult(DDTraceId.from(123), 456, true, false, true, false, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalB = new ModuleExecutionResult(DDTraceId.from(234), 567, false, true, false, true, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalA = new ModuleExecutionResult(DDTraceId.from(123), 456, true, false, true, false, true, false, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signalB = new ModuleExecutionResult(DDTraceId.from(234), 567, false, true, false, true, false, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def server = new SignalServer()
def received = new ArrayList()

Expand Down Expand Up @@ -118,7 +118,7 @@ class SignalServerTest extends Specification {
when:
def address = server.getAddress()
try (def client = new SignalClient(address, clientTimeoutMillis)) {
client.send(new ModuleExecutionResult(DDTraceId.from(123), 456, false, false, false, false, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2"))))
client.send(new ModuleExecutionResult(DDTraceId.from(123), 456, false, false, false, false, false, false, 0, Collections.singletonList(new TestFramework("junit", "4.13.2"))))
}

then:
Expand All @@ -130,7 +130,7 @@ class SignalServerTest extends Specification {

def "test error response receipt"() {
given:
def signal = new ModuleExecutionResult(DDTraceId.from(123), 456, true, true, false, false, true, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def signal = new ModuleExecutionResult(DDTraceId.from(123), 456, true, true, false, false, true, false, 1, Collections.singletonList(new TestFramework("junit", "4.13.2")))
def server = new SignalServer()

def errorResponse = new ErrorResponse("An error occurred while processing the signal")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_6},
"_dd.test.has_failed_test_replay" : "true",
"_dd.test.is_user_provided_service" : "true",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"ci.workspace_path" : ${content_meta_ci_workspace_path},
Expand Down Expand Up @@ -371,6 +372,7 @@
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_7},
"_dd.test.has_failed_test_replay" : "true",
"_dd.test.is_user_provided_service" : "true",
"ci.workspace_path" : ${content_meta_ci_workspace_path},
"component" : "junit5",
Expand Down
Loading