Skip to content

Try fix flake in OTLP logs tests #8508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andrewlock merged 2 commits into from
Apr 28, 2026
+78 −26
Merged

Try fix flake in OTLP logs tests #8508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
30ce431
Try to fix otel flakiness
andrewlock Apr 23, 2026
b4b2abd
Reduce Flake in OTLP logs tests
andrewlock Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 53 additions & 1 deletion tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ public async Task SubmitsOtlpLogs(string packageVersion, string datadogLogsEnabl
SetEnvironmentVariable("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
// Short delay gives the OTel SDK multiple periodic exports before LoggerProviderSdk.Dispose() hits its 5s shutdown timeout.
// This is especially important for gRPC, where the first export warms the HTTP/2 connection.
SetEnvironmentVariable("OTEL_BLRP_SCHEDULE_DELAY", "500");
SetEnvironmentVariable("OTEL_BLRP_SCHEDULE_DELAY", "100");
SetEnvironmentVariable("DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL", "Verbose");

if (useAgentHostBackup)
Expand Down Expand Up @@ -698,6 +698,58 @@ public async Task SubmitsOtlpLogs(string packageVersion, string datadogLogsEnabl
}
}

// The OTel SDK can emit multiple OTLP batches during the sample run (periodic export(s) plus shutdown flush),
// and the test-agent can return them as separate top-level elements in any order. Collapse everything
// into a single batch, then sort scope_logs by scope.name and log_records by body.string_value so the
// snapshot is stable regardless of batch split, arrival order, or the exporter's internal scope grouping.
if (logsData is JArray logsArray && logsArray.Count > 1)
{
// Add all the subsequent logs to the first array
var mergedScopeLogs = (JArray)logsArray[0]["resource_logs"][0]["scope_logs"];
for (var i = 1; i < logsArray.Count; i++)
{
foreach (var scopeLog in (JArray)logsArray[i]["resource_logs"][0]["scope_logs"])
{
var scopeName = scopeLog["scope"]?["name"]?.ToString();
var existing = mergedScopeLogs.FirstOrDefault(s => s["scope"]?["name"]?.ToString() == scopeName);
if (existing != null && scopeLog["log_records"] is JArray incoming)
{
var existingRecords = (JArray)existing["log_records"];
foreach (var record in incoming)
{
existingRecords.Add(record);
}
}
else
{
mergedScopeLogs.Add(scopeLog);
}
}
}

while (logsArray.Count > 1)
{
logsArray.RemoveAt(1);
}
}

// Fix the ordering to ensure it's deterministic
foreach (var resourceLog in logsData.SelectTokens("$[0].resource_logs[*]"))
{
if (resourceLog["scope_logs"] is JArray scopeLogsArray)
{
foreach (var scopeLog in scopeLogsArray)
{
if (scopeLog["log_records"] is JArray recordsArray)
{
scopeLog["log_records"] = new JArray(recordsArray.OrderBy(r => r["body"]?["string_value"]?.ToString()));
}
}

resourceLog["scope_logs"] = new JArray(scopeLogsArray.OrderBy(s => s["scope"]?["name"]?.ToString()));
}
}

var formattedJson = logsData.ToString(Formatting.Indented);
var settings = VerifyHelper.GetSpanVerifierSettings();
var suffix = GetSuffix(packageVersion);
Expand Down
50 changes: 25 additions & 25 deletions tracer/test/snapshots/OpenTelemetrySdkTests.SubmitsOtlpLogs.verified.txt
View file
Open in desktop
Copy link
Copy Markdown
Member Author

@andrewlock andrewlock Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The changes here are just re-ordering the logs to be in body.string_value order

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[
[
{
"resource_logs": [
{
Expand Down Expand Up @@ -42,6 +42,15 @@
"name": "LogServiceName"
},
"log_records": [
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_INFO",
"severity_text": "Information",
"body": {
"string_value": "Hello from OpenTelemetry logger outside span context"
},
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_INFO",
Expand All @@ -53,6 +62,15 @@
"span_id": "normalized-span-id",
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_FATAL",
"severity_text": "Critical",
"body": {
"string_value": "This is a critical message"
},
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_TRACE",
Expand Down Expand Up @@ -85,24 +103,6 @@
"trace_id": "normalized-trace-id",
"span_id": "normalized-span-id",
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_INFO",
"severity_text": "Information",
"body": {
"string_value": "Hello from OpenTelemetry logger outside span context"
},
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_FATAL",
"severity_text": "Critical",
"body": {
"string_value": "This is a critical message"
},
"observed_time_unix_nano": "0"
}
]
},
Expand All @@ -113,21 +113,21 @@
"log_records": [
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_INFO",
"severity_text": "Information",
"severity_number": "SEVERITY_NUMBER_DEBUG",
"severity_text": "Debug",
"body": {
"string_value": "Response from other library"
"string_value": "Debug information from other library"
},
"trace_id": "normalized-trace-id",
"span_id": "normalized-span-id",
"observed_time_unix_nano": "0"
},
{
"time_unix_nano": "0",
"severity_number": "SEVERITY_NUMBER_DEBUG",
"severity_text": "Debug",
"severity_number": "SEVERITY_NUMBER_INFO",
"severity_text": "Information",
"body": {
"string_value": "Debug information from other library"
"string_value": "Response from other library"
},
"trace_id": "normalized-trace-id",
"span_id": "normalized-span-id",
Expand Down
Loading