diff --git a/Datadog.Trace.OSX.slnf b/Datadog.Trace.OSX.slnf index 6dd789d79de0..a5e679fd98ad 100644 --- a/Datadog.Trace.OSX.slnf +++ b/Datadog.Trace.OSX.slnf @@ -90,8 +90,8 @@ "tracer\\test\\test-applications\\integrations\\Samples.AzureServiceBus\\Samples.AzureServiceBus.csproj", "tracer\\test\\test-applications\\integrations\\Samples.CIVisibilityIpc\\Samples.CIVisibilityIpc.csproj", "tracer\\test\\test-applications\\integrations\\Samples.Console\\Samples.Console.csproj", - "tracer\\test\\test-applications\\integrations\\Samples.CosmosDb\\Samples.CosmosDb.csproj", "tracer\\test\\test-applications\\integrations\\Samples.CosmosDb.Vnext\\Samples.CosmosDb.Vnext.csproj", + "tracer\\test\\test-applications\\integrations\\Samples.CosmosDb\\Samples.CosmosDb.csproj", "tracer\\test\\test-applications\\integrations\\Samples.Couchbase3\\Samples.Couchbase3.csproj", "tracer\\test\\test-applications\\integrations\\Samples.Couchbase\\Samples.Couchbase.csproj", "tracer\\test\\test-applications\\integrations\\Samples.Dapper\\Samples.Dapper.csproj", @@ -181,7 +181,6 @@ "tracer\\test\\test-applications\\regression\\Sandbox.ManualTracing\\Sandbox.ManualTracing.csproj", "tracer\\test\\test-applications\\regression\\ServiceBus.Minimal.MassTransit\\ServiceBus.Minimal.MassTransit.csproj", "tracer\\test\\test-applications\\regression\\ServiceBus.Minimal.Rebus\\ServiceBus.Minimal.Rebus.csproj", - "tracer\\test\\test-applications\\regression\\StackExchange.Redis.AssemblyConflict.LegacyProject\\StackExchange.Redis.AssemblyConflict.LegacyProject.csproj", "tracer\\test\\test-applications\\regression\\StackExchange.Redis.AssemblyConflict.SdkProject\\StackExchange.Redis.AssemblyConflict.SdkProject.csproj", "tracer\\test\\test-applications\\regression\\StackExchange.Redis.StackOverflowException\\StackExchange.Redis.StackOverflowException.csproj", "tracer\\test\\test-applications\\Samples.Shared\\Samples.Shared.shproj", diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpFieldNames.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpFieldNames.cs new file mode 100644 index 000000000000..5278baebd59c --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpFieldNames.cs @@ -0,0 +1,46 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.Helpers +{ + /// + /// The test agent renders an OTLP http/json payload with camelCase field names and an + /// http/protobuf payload with snake_case field names. This maps a protocol to the casing + /// used when walking the rendered JSON. + /// + internal readonly struct OtlpFieldNames + { + private OtlpFieldNames(bool isJson) + { + IsJson = isJson; + } + + public bool IsJson { get; } + + public string ResourceSpans => IsJson ? "resourceSpans" : "resource_spans"; + + public string ScopeSpans => IsJson ? "scopeSpans" : "scope_spans"; + + public string StringValue => IsJson ? "stringValue" : "string_value"; + + public string IntValue => IsJson ? "intValue" : "int_value"; + + public string TraceId => IsJson ? "traceId" : "trace_id"; + + public string SpanId => IsJson ? "spanId" : "span_id"; + + public string ParentSpanId => IsJson ? "parentSpanId" : "parent_span_id"; + + public string StartTimeUnixNano => IsJson ? "startTimeUnixNano" : "start_time_unix_nano"; + + public string EndTimeUnixNano => IsJson ? "endTimeUnixNano" : "end_time_unix_nano"; + + public string TimeUnixNano => IsJson ? "timeUnixNano" : "time_unix_nano"; + + public static OtlpFieldNames For(bool isJson) => new(isJson); + } +} diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpSnapshotHelper.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpSnapshotHelper.cs new file mode 100644 index 000000000000..47a31185024d --- /dev/null +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/OtlpSnapshotHelper.cs @@ -0,0 +1,449 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Datadog.Trace.TestHelpers; +using Datadog.Trace.Vendors.Newtonsoft.Json.Linq; +using FluentAssertions; +using VerifyTests; + +namespace Datadog.Trace.ClrProfiler.IntegrationTests.Helpers +{ + /// + /// Shared plumbing for tests that snapshot OTLP payloads captured by the ddapm test agent. + /// + internal static class OtlpSnapshotHelper + { + // Single source of truth for translating an OTLP http/protobuf payload (rendered as JSON + // by the test agent with snake_case field names and string-form enum values) to the + // OTLP http/json shape (camelCase field names, integer enum values). When a new OTLP + // field or enum reaches the serializer, add the mapping here. + private static readonly (string From, string To)[] ProtobufToJsonFieldNameMappings = + { + ("\"resource_spans\"", "\"resourceSpans\""), + ("\"scope_spans\"", "\"scopeSpans\""), + ("\"trace_id\"", "\"traceId\""), + ("\"span_id\"", "\"spanId\""), + ("\"parent_span_id\"", "\"parentSpanId\""), + ("\"start_time_unix_nano\"", "\"startTimeUnixNano\""), + ("\"end_time_unix_nano\"", "\"endTimeUnixNano\""), + ("\"time_unix_nano\"", "\"timeUnixNano\""), + ("\"string_value\"", "\"stringValue\""), + ("\"double_value\"", "\"doubleValue\""), + ("\"int_value\"", "\"intValue\""), + ("\"bool_value\"", "\"boolValue\""), + ("\"array_value\"", "\"arrayValue\""), + }; + + private static readonly (string From, string To)[] ProtobufToJsonEnumMappings = + { + ("\"kind\": \"SPAN_KIND_INTERNAL\"", "\"kind\": 1"), + ("\"kind\": \"SPAN_KIND_SERVER\"", "\"kind\": 2"), + ("\"kind\": \"SPAN_KIND_CLIENT\"", "\"kind\": 3"), + ("\"kind\": \"SPAN_KIND_PRODUCER\"", "\"kind\": 4"), + ("\"kind\": \"SPAN_KIND_CONSUMER\"", "\"kind\": 5"), + ("\"code\": \"STATUS_CODE_UNSET\"", "\"code\": 0"), + ("\"code\": \"STATUS_CODE_OK\"", "\"code\": 1"), + ("\"code\": \"STATUS_CODE_ERROR\"", "\"code\": 2"), + }; + + private static readonly Regex TraceIdRegex = new(@"^([a-fA-F0-9]{32})$"); + + private static readonly Regex SpanIdRegex = new(@"^([a-fA-F0-9]{16})$"); + + public static void AddProtobufToJsonScrubbers(VerifySettings settings) + { + foreach (var (from, to) in ProtobufToJsonFieldNameMappings) + { + settings.AddSimpleScrubber(from, to); + } + + foreach (var (from, to) in ProtobufToJsonEnumMappings) + { + settings.AddSimpleScrubber(from, to); + } + } + + /// + /// Clears the test-agent session, retrying if the agent is not yet ready. + /// Ensures the OTLP HTTP endpoint is accepting connections before tests proceed. + /// + /// The host the test agent is listening on. + /// The number of attempts to make before failing. + /// The delay between attempts, in milliseconds. + /// A task that completes once the session has been cleared. + public static async Task ClearTestAgentSessionAsync(string testAgentHost, int maxRetries = 5, int delayMs = 1000) + { + using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(5) }; + var url = $"http://{testAgentHost}:4318/test/session/clear"; + + for (var attempt = 1; attempt <= maxRetries; attempt++) + { + try + { + var response = await httpClient.GetAsync(url); + response.EnsureSuccessStatusCode(); + return; + } + catch (Exception) when (attempt < maxRetries) + { + await Task.Delay(delayMs); + } + } + + // Final attempt -- let it throw if it fails + var finalResponse = await httpClient.GetAsync(url); + finalResponse.EnsureSuccessStatusCode(); + } + + /// + /// Polls the test-agent for data until non-empty results are returned or timeout is reached. + /// The sample app exports data during shutdown, so there can be a brief delay + /// between process exit and data appearing in the test-agent. The timeout is generous + /// because first-time gRPC connections (TCP+HTTP/2+TLS handshake) plus tracer shutdown + /// flushing can stack up on slower CI runners. + /// + /// The test agent endpoint to poll. + /// How long to keep polling before giving up. + /// The delay between polls, in milliseconds. + /// The data returned by the test agent. + public static async Task WaitForTestAgentDataAsync(string url, int timeoutSeconds = 60, int pollIntervalMs = 500) + { + using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; + var deadline = DateTime.UtcNow.AddSeconds(timeoutSeconds); + + while (DateTime.UtcNow < deadline) + { + var response = await httpClient.GetAsync(url); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var data = JToken.Parse(json); + + if (data.HasValues) + { + return data; + } + + await Task.Delay(pollIntervalMs); + } + + // Final attempt -- return whatever we get so the caller's assertion shows the actual value + var finalResponse = await httpClient.GetAsync(url); + finalResponse.EnsureSuccessStatusCode(); + var finalJson = await finalResponse.Content.ReadAsStringAsync(); + return JToken.Parse(finalJson); + } + + /// + /// Replaces the resource attributes that change between runs or between machines. + /// + /// The captured OTLP requests. + /// The field-name casing to use. + public static void NormalizeResourceAttributes(JToken tracesRequests, OtlpFieldNames names) + { + var stringValueKey = names.StringValue; + + foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.version')]")) + { + attribute["value"]![stringValueKey] = "sdk-version"; + } + + foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.name')]")) + { + attribute["value"]![stringValueKey] = "sdk-name"; + } + + foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'git.commit.sha')]")) + { + attribute["value"]![stringValueKey] = "normalized-git-commit-sha"; + } + } + + /// + /// Asserts that the trace ids, span ids, and timestamps are well-formed, then replaces them + /// with fixed placeholders so the payload is stable across runs. + /// + /// The captured OTLP requests. + /// The field-name casing to use. + /// The time the sample application was started, used as a lower bound for span timestamps. + public static void NormalizeSpans(JToken tracesRequests, OtlpFieldNames names, long applicationStartTimeUnixNano) + { + var isJson = names.IsJson; + var stringValueKey = names.StringValue; + var traceIdKey = names.TraceId; + var spanIdKey = names.SpanId; + var parentSpanIdKey = names.ParentSpanId; + var startTimeUnixNanoKey = names.StartTimeUnixNano; + var endTimeUnixNanoKey = names.EndTimeUnixNano; + var timeUnixNanoKey = names.TimeUnixNano; + + foreach (var span in tracesRequests.SelectTokens("$..spans[*]")) + { + // Parse unstable information from the span + string traceIdData = isJson ? span[traceIdKey]!.ToString() + : ToTraceId(Convert.FromBase64String(span[traceIdKey]!.ToString())); + string spanIdData = isJson ? span[spanIdKey]!.ToString() + : ToSpanId(Convert.FromBase64String(span[spanIdKey]!.ToString())); + var spanStartTimeUnixNano = long.Parse(span[startTimeUnixNanoKey]!.ToString()); + var spanEndTimeUnixNano = long.Parse(span[endTimeUnixNanoKey]!.ToString()); + + // Add strong assertions on unstable span information + spanStartTimeUnixNano.Should().BeGreaterThanOrEqualTo(applicationStartTimeUnixNano); + spanEndTimeUnixNano.Should().BeGreaterThanOrEqualTo(spanStartTimeUnixNano); + traceIdData.Should().MatchRegex(TraceIdRegex); + spanIdData.Should().MatchRegex(SpanIdRegex); + if (span[parentSpanIdKey] is not null) + { + string? parentSpanIdData = isJson ? span[parentSpanIdKey]?.ToString() + : ToSpanId(Convert.FromBase64String(span[parentSpanIdKey]!.ToString())); + parentSpanIdData.Should().MatchRegex(SpanIdRegex); + } + + // Normalize the unstable span information for our snapshots + span[startTimeUnixNanoKey] = "0"; + span[endTimeUnixNanoKey] = "0"; + span[traceIdKey] = "normalized-trace-id"; + span[spanIdKey] = "normalized-span-id"; + if (span[parentSpanIdKey] is not null) + { + span[parentSpanIdKey] = "normalized-parent-span-id"; + } + + // Our JSON and Protobuf OTLP exporters differ in serialization behavior when there are no attributes. + // Standardize them here by removing an empty array + if (span["attributes"] is JArray attributes && attributes.Count == 0) + { + ((JObject)span).Remove("attributes"); + } + } + + foreach (var attribute in tracesRequests.SelectTokens("$..spans[*].attributes[?(@.key == 'otel.trace_id')]")) + { + attribute["value"]![stringValueKey] = "normalized-otel-trace-id"; + } + + foreach (var link in tracesRequests.SelectTokens("$..links[*]")) + { + if (isJson) + { + link[traceIdKey]!.ToString().Should().MatchRegex(TraceIdRegex); + link[spanIdKey]!.ToString().Should().MatchRegex(SpanIdRegex); + } + + link[traceIdKey] = "normalized-trace-id"; + link[spanIdKey] = "normalized-span-id"; + } + + foreach (var @event in tracesRequests.SelectTokens("$..events[*]")) + { + ((JObject)@event).Remove(timeUnixNanoKey); + ((JObject)@event).AddFirst(new JProperty(timeUnixNanoKey, "0")); + } + } + + /// + /// Collapses every captured request into the first one. Asserts first that each request + /// carries identical resource attributes and a single instrumentation scope, which holds for + /// the Datadog SDK because it emits one application-level resource and does not yet track + /// spans per instrumentation scope. + /// + /// The captured OTLP requests. + /// The field-name casing to use. + /// Orders the merged spans. Defaults to ordering by span name. + /// The single merged request. + public static JToken MergeDatadogRequests( + JToken tracesRequests, + OtlpFieldNames names, + Func, IEnumerable>? sortSpans = null) + { + var resourceSpansKey = names.ResourceSpans; + var scopeSpansKey = names.ScopeSpans; + + // First, for the DD SDK, assert that the resource attributes for all requests are identical + // This is analogous to DD_SERVICE, DD_VERSION, DD_ENV, etc. that define + // metadata for the telemetry at an application and host level. + // This is different for OTel SDK application since the in-app code uses the SDK to create a + // 2nd, completely distinct, Traces SDK instance + JToken? previousResourceAttributes = null; + foreach (var tracesRequest in tracesRequests) + { + tracesRequest[resourceSpansKey].Should().HaveCount(1); + var resourceAttributes = tracesRequest[resourceSpansKey]![0]!["resource"]!["attributes"]; + + if (previousResourceAttributes is null) + { + previousResourceAttributes = resourceAttributes; + } + else + { + JToken.DeepEquals(previousResourceAttributes, resourceAttributes).Should().BeTrue(); + previousResourceAttributes = resourceAttributes; + } + } + + // Next, assert that we only have a singular InstrumentationScope in each request. + // In OpenTelemetry, an InstrumentationScope is a way to group spans by the library that produced them. + // We should be respecting this for each library/ActivitySource, but right now the DD SDK doesn't + // keep track of that information, so consolidate them into one single, empty InstrumentationScope. + // TODO: Properly track spans per instrumentation scope. + JArray? firstSpans = null; + foreach (var tracesRequest in tracesRequests) + { + tracesRequest[resourceSpansKey]![0]![scopeSpansKey].Should().HaveCount(1); + var spans = tracesRequest[resourceSpansKey]![0]![scopeSpansKey]![0]!["spans"] as JArray; + + if (firstSpans is null) + { + firstSpans = spans; + } + else + { + foreach (var span in spans!) + { + firstSpans.Add(span); + } + } + } + + // Now re-order and trim down to one single request + // This means the output is not a true 1:1 mapping of the input spans, but it's good enough for now + // and will make the results stable. + // Also, sort the spans by name to stabilize + sortSpans ??= spans => spans.OrderBy(s => s["name"]!.ToString()); + var sortedSpans = new JArray(sortSpans(firstSpans!)); + tracesRequests[0]![resourceSpansKey]![0]![scopeSpansKey]![0]!["spans"] = sortedSpans; + return tracesRequests[0]!; + } + + /// + /// Sorts spans by name within each scope, leaving the request structure intact. Used when the + /// payload comes from a real OTel SDK, which emits genuinely distinct instrumentation scopes. + /// + /// The captured OTLP requests. + /// The field-name casing to use. + public static void SortSpansPerScope(JToken tracesRequests, OtlpFieldNames names) + { + foreach (var scopeSpan in tracesRequests.SelectTokens($"$..{names.ScopeSpans}[*]")) + { + if (scopeSpan["spans"] is JArray spansArray) + { + var sorted = new JArray(spansArray.OrderBy(s => s["name"]?.ToString())); + scopeSpan["spans"] = sorted; + } + } + } + + /// + /// Returns the string value of the first attribute matching any of , + /// or null when the span carries none of them. Accepts several keys because a tag's name + /// changes with the semantic conventions in play, for example http.url versus url.full. + /// + /// The span to read from. + /// The field-name casing to use. + /// The attribute keys to look for, in priority order. + /// The attribute value, or null when the span has none of the keys. + public static string? GetAttributeStringValue(JToken span, OtlpFieldNames names, params string[] keys) + { + if (span["attributes"] is not JArray attributes) + { + return null; + } + + foreach (var key in keys) + { + foreach (var attribute in attributes) + { + if (attribute["key"]?.ToString() == key) + { + return attribute["value"]?[names.StringValue]?.ToString(); + } + } + } + + return null; + } + + /// + /// Sets a string attribute on a span, appending it when the span does not already have it. + /// + /// The span to modify. + /// The field-name casing to use. + /// The attribute key. + /// The attribute value. + public static void SetAttributeStringValue(JToken span, OtlpFieldNames names, string key, string value) + { + if (span["attributes"] is not JArray attributes) + { + attributes = new JArray(); + ((JObject)span)["attributes"] = attributes; + } + + foreach (var attribute in attributes) + { + if (attribute["key"]?.ToString() == key) + { + attribute["value"] = new JObject { [names.StringValue] = value }; + return; + } + } + + attributes.Add(new JObject + { + ["key"] = key, + ["value"] = new JObject { [names.StringValue] = value }, + }); + } + + /// + /// Sorts every span's attribute array by key. Attribute order otherwise follows tag + /// enumeration order, which is not guaranteed to be stable across runtimes. + /// + /// The captured OTLP requests. + public static void SortSpanAttributes(JToken tracesRequests) + { + foreach (var span in tracesRequests.SelectTokens("$..spans[*]")) + { + if (span["attributes"] is JArray attributes) + { + ((JObject)span)["attributes"] = new JArray( + attributes.OrderBy(a => a["key"]?.ToString() ?? string.Empty, StringComparer.Ordinal)); + } + } + } + + private static string ToHexString(byte[] bytes, int length) + { + bytes.Length.Should().Be(length); + + var traceId = new byte[length * 2]; + for (int i = 0; i < length; i++) + { + traceId[2 * i] = (byte)(bytes[i] >> 4); // high 4 bits + traceId[(2 * i) + 1] = (byte)(bytes[i] & 0x0F); // low 4 bits + } + + // Convert each nibble (0-15) to its hex character + var result = new char[length * 2]; + for (int i = 0; i < length * 2; i++) + { + result[i] = (char)(traceId[i] < 10 ? '0' + traceId[i] : 'a' + traceId[i] - 10); + } + + return new string(result); + } + + private static string ToTraceId(byte[] bytes) => ToHexString(bytes, 16); + + private static string ToSpanId(byte[] bytes) => ToHexString(bytes, 8); + } +} diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs index 65ef4c751fed..e274cac22fc8 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs @@ -6,9 +6,9 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers; using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.TestHelpers; @@ -76,46 +76,11 @@ public class OpenTelemetrySdkTests : TracingIntegrationTest "network.protocol.name" }; - // Single source of truth for translating an OTLP http/protobuf payload (rendered as JSON - // by the test agent with snake_case field names and string-form enum values) to the - // OTLP http/json shape (camelCase field names, integer enum values). When a new OTLP - // field or enum reaches the serializer, add the mapping here. - private static readonly (string From, string To)[] ProtobufToJsonFieldNameMappings = - { - ("\"resource_spans\"", "\"resourceSpans\""), - ("\"scope_spans\"", "\"scopeSpans\""), - ("\"trace_id\"", "\"traceId\""), - ("\"span_id\"", "\"spanId\""), - ("\"parent_span_id\"", "\"parentSpanId\""), - ("\"start_time_unix_nano\"", "\"startTimeUnixNano\""), - ("\"end_time_unix_nano\"", "\"endTimeUnixNano\""), - ("\"time_unix_nano\"", "\"timeUnixNano\""), - ("\"string_value\"", "\"stringValue\""), - ("\"double_value\"", "\"doubleValue\""), - ("\"int_value\"", "\"intValue\""), - ("\"bool_value\"", "\"boolValue\""), - ("\"array_value\"", "\"arrayValue\""), - }; - - private static readonly (string From, string To)[] ProtobufToJsonEnumMappings = - { - ("\"kind\": \"SPAN_KIND_INTERNAL\"", "\"kind\": 1"), - ("\"kind\": \"SPAN_KIND_SERVER\"", "\"kind\": 2"), - ("\"kind\": \"SPAN_KIND_CLIENT\"", "\"kind\": 3"), - ("\"kind\": \"SPAN_KIND_PRODUCER\"", "\"kind\": 4"), - ("\"kind\": \"SPAN_KIND_CONSUMER\"", "\"kind\": 5"), - ("\"code\": \"STATUS_CODE_UNSET\"", "\"code\": 0"), - ("\"code\": \"STATUS_CODE_OK\"", "\"code\": 1"), - ("\"code\": \"STATUS_CODE_ERROR\"", "\"code\": 2"), - }; - private readonly Regex _versionRegex = new(@"telemetry.sdk.version: (0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)"); private readonly Regex _timeUnixNanoRegex = new(@"time_unix_nano"":([0-9]{10}[0-9]+)"); private readonly Regex _exceptionStacktraceRegex = new(@"exception.stacktrace"":""System.ArgumentException: Example argument exception.*"","""); private readonly Regex _exceptionStacktraceOtlpRegex = new(@"string_value"": ""System.ArgumentException: Example argument exception.*"""); private readonly Regex _exceptionStacktraceOtlpJsonRegex = new(@"stringValue"": ""System.ArgumentException: Example argument exception.*"""); - private readonly Regex _traceIdRegex = new(@"^([a-fA-F0-9]{32})$"); - private readonly Regex _spanIdRegex = new(@"^([a-fA-F0-9]{16})$"); public OpenTelemetrySdkTests(ITestOutputHelper output) : base("OpenTelemetrySdk", output) @@ -294,7 +259,7 @@ public async Task SubmitsOtlpTraces(string packageVersion, string datadogTracesE var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "127.0.0.1"; var otlpPort = protocol == "grpc" ? 4317 : 4318; - await ClearTestAgentSession(testAgentHost); + await OtlpSnapshotHelper.ClearTestAgentSessionAsync(testAgentHost); // This is the key configuration that is set differently from previous test cases: // OTEL_TRACES_EXPORTER=otlp enables the DD SDK to emit traces (and trace stats) via OTLP @@ -334,203 +299,25 @@ public async Task SubmitsOtlpTraces(string packageVersion, string datadogTracesE // between process exit and the data appearing in the test-agent. Poll with // retries to avoid a race, matching the pattern used by SubmitsOtlpMetrics // and SubmitsOtlpLogs. - var tracesRequests = await WaitForTestAgentData($"http://{testAgentHost}:4318/test/session/traces"); + var tracesRequests = await OtlpSnapshotHelper.WaitForTestAgentDataAsync($"http://{testAgentHost}:4318/test/session/traces"); tracesRequests.Should().NotBeNullOrEmpty(); // Normalize the data in resource attributes and spans - var resourceSpansKey = isJson ? "resourceSpans" : "resource_spans"; - var scopeSpansKey = isJson ? "scopeSpans" : "scope_spans"; - var stringValueKey = isJson ? "stringValue" : "string_value"; - var traceIdKey = isJson ? "traceId" : "trace_id"; - var spanIdKey = isJson ? "spanId" : "span_id"; - var parentSpanIdKey = isJson ? "parentSpanId" : "parent_span_id"; - var startTimeUnixNanoKey = isJson ? "startTimeUnixNano" : "start_time_unix_nano"; - var endTimeUnixNanoKey = isJson ? "endTimeUnixNano" : "end_time_unix_nano"; - var timeUnixNanoKey = isJson ? "timeUnixNano" : "time_unix_nano"; - - foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.version')]")) - { - attribute["value"]![stringValueKey] = "sdk-version"; - } - - foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.name')]")) - { - attribute["value"]![stringValueKey] = "sdk-name"; - } - - foreach (var attribute in tracesRequests.SelectTokens("$..resource.attributes[?(@.key == 'git.commit.sha')]")) - { - attribute["value"]![stringValueKey] = "normalized-git-commit-sha"; - } - - foreach (var span in tracesRequests.SelectTokens("$..spans[*]")) - { - static string ToHexString(byte[] bytes, int length) - { - bytes.Length.Should().Be(length); - - var traceId = new byte[length * 2]; - for (int i = 0; i < length; i++) - { - traceId[2 * i] = (byte)(bytes[i] >> 4); // high 4 bits - traceId[(2 * i) + 1] = (byte)(bytes[i] & 0x0F); // low 4 bits - } - - // Convert each nibble (0-15) to its hex character - var result = new char[length * 2]; - for (int i = 0; i < length * 2; i++) - { - result[i] = (char)(traceId[i] < 10 ? '0' + traceId[i] : 'a' + traceId[i] - 10); - } - - return new string(result); - } - - static string ToTraceId(byte[] bytes) => ToHexString(bytes, 16); - - static string ToSpanId(byte[] bytes) => ToHexString(bytes, 8); - - // Parse unstable information from the span - string traceIdData = isJson ? span[traceIdKey].ToString() - : ToTraceId(Convert.FromBase64String(span[traceIdKey].ToString())); - string spanIdData = isJson ? span[spanIdKey].ToString() - : ToSpanId(Convert.FromBase64String(span[spanIdKey].ToString())); - var spanStartTimeUnixNano = long.Parse(span[startTimeUnixNanoKey].ToString()); - var spanEndTimeUnixNano = long.Parse(span[endTimeUnixNanoKey].ToString()); - - // Add strong assertions on unstable span information - spanStartTimeUnixNano.Should().BeGreaterThanOrEqualTo(applicationStartTimeUnixNano); - spanEndTimeUnixNano.Should().BeGreaterThanOrEqualTo(spanStartTimeUnixNano); - traceIdData.Should().MatchRegex(_traceIdRegex); - spanIdData.Should().MatchRegex(_spanIdRegex); - if (span[parentSpanIdKey] != null) - { - string parentSpanIdData = isJson ? span[parentSpanIdKey]?.ToString() - : ToSpanId(Convert.FromBase64String(span[parentSpanIdKey].ToString())); - parentSpanIdData.Should().MatchRegex(_spanIdRegex); - } - - // Normalize the unstable span information for our snapshots - span[startTimeUnixNanoKey] = "0"; - span[endTimeUnixNanoKey] = "0"; - span[traceIdKey] = "normalized-trace-id"; - span[spanIdKey] = "normalized-span-id"; - if (span[parentSpanIdKey] != null) - { - span[parentSpanIdKey] = "normalized-parent-span-id"; - } - - // Our JSON and Protobuf OTLP exporters differ in serialization behavior when there are no attributes. - // Standardize them here by removing an empty array - if (span["attributes"] is JArray attributes && attributes.Count == 0) - { - ((JObject)span).Remove("attributes"); - } - } - - foreach (var attribute in tracesRequests.SelectTokens("$..spans[*].attributes[?(@.key == 'otel.trace_id')]")) - { - attribute["value"]![stringValueKey] = "normalized-otel-trace-id"; - } - - foreach (var link in tracesRequests.SelectTokens("$..links[*]")) - { - if (isJson) - { - link[traceIdKey].ToString().Should().MatchRegex(_traceIdRegex); - link[spanIdKey].ToString().Should().MatchRegex(_spanIdRegex); - } - else - { - // We need to emit each byte as a character, so use ASCII encoding - // var decodedTraceId = System.Text.Encoding.ASCII.GetString(Convert.FromBase64String(link[traceIdKey].ToString())); - // var decodedSpanId = System.Text.Encoding.ASCII.GetString(Convert.FromBase64String(link[spanIdKey].ToString())); - // decodedTraceId.Should().MatchRegex(_traceIdRegex); - // decodedSpanId.Should().MatchRegex(_spanIdRegex); - } - - link[traceIdKey] = "normalized-trace-id"; - link[spanIdKey] = "normalized-span-id"; - } - - foreach (var @event in tracesRequests.SelectTokens("$..events[*]")) - { - ((JObject)@event).Remove(timeUnixNanoKey); - ((JObject)@event).AddFirst(new JProperty(timeUnixNanoKey, "0")); - } + var names = OtlpFieldNames.For(isJson); + OtlpSnapshotHelper.NormalizeResourceAttributes(tracesRequests, names); + OtlpSnapshotHelper.NormalizeSpans(tracesRequests, names, applicationStartTimeUnixNano); // For the Datadog SDK, perform more sanitization string finalJson; if (datadogTracesEnabled.Equals("true")) { - // First, for the DD SDK, assert that the resource attributes for all requests are identical - // This is analogous to DD_SERVICE, DD_VERSION, DD_ENV, etc. that define - // metadata for the telemetry at an application and host level. - // This is different for OTel SDK application since the in-app code uses the SDK to create a - // 2nd, completely distinct, Traces SDK instance - - JToken previousResourceAttributes = null; - foreach (var tracesRequest in tracesRequests) - { - tracesRequest[resourceSpansKey].Should().HaveCount(1); - var resourceAttributes = tracesRequest[resourceSpansKey][0]["resource"]["attributes"]; - - if (previousResourceAttributes == null) - { - previousResourceAttributes = resourceAttributes; - } - else - { - JToken.DeepEquals(previousResourceAttributes, resourceAttributes).Should().BeTrue(); - previousResourceAttributes = resourceAttributes; - } - } - - // Next, assert that we only have a singular InstrumentationScope in each request. - // In OpenTelemetry, an InstrumentationScope is a way to group spans by the library that produced them. - // We should be respecting this for each library/ActivitySource, but right now the DD SDK doesn't - // keep track of that information, so consolidate them into one single, empty InstrumentationScope. - // TODO: Properly track spans per instrumentation scope. - JArray firstSpans = null; - foreach (var tracesRequest in tracesRequests) - { - tracesRequest[resourceSpansKey][0][scopeSpansKey].Should().HaveCount(1); - var spans = tracesRequest[resourceSpansKey][0][scopeSpansKey][0]["spans"] as JArray; - - if (firstSpans == null) - { - firstSpans = spans; - } - else - { - foreach (var span in spans) - { - firstSpans.Add(span); - } - } - } - - // Now re-order and trim down to one single request - // This means the output is not a true 1:1 mapping of the input spans, but it's good enough for now - // and will make the results stable. - // Also, sort the spans by name to stabilize - var sortedSpans = new JArray(firstSpans.OrderBy(s => s["name"]!.ToString())); - tracesRequests[0][resourceSpansKey][0][scopeSpansKey][0]["spans"] = sortedSpans; - finalJson = tracesRequests[0].ToString(Formatting.Indented); + finalJson = OtlpSnapshotHelper.MergeDatadogRequests(tracesRequests, names) + .ToString(Formatting.Indented); } else { - // Sort the spans by name to stabilize - foreach (var scopeSpan in tracesRequests.SelectTokens($"$..{scopeSpansKey}[*]")) - { - if (scopeSpan["spans"] is JArray spansArray) - { - var sorted = new JArray(spansArray.OrderBy(s => s["name"]?.ToString())); - scopeSpan["spans"] = sorted; - } - } - + OtlpSnapshotHelper.SortSpansPerScope(tracesRequests, names); finalJson = tracesRequests.ToString(Formatting.Indented); } @@ -541,7 +328,7 @@ static string ToHexString(byte[] bytes, int length) // Add scrubbers only for http/protobuf if (protocol == "http/protobuf") { - AddProtobufToJsonScrubbers(settings); + OtlpSnapshotHelper.AddProtobufToJsonScrubbers(settings); } var fileName = $"{nameof(OpenTelemetrySdkTests)}.SubmitsOtlpTraces{snapshotName}"; @@ -574,7 +361,7 @@ public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetric var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "localhost"; var otlpPort = protocol == "grpc" ? 4317 : 4318; - await ClearTestAgentSession(testAgentHost); + await OtlpSnapshotHelper.ClearTestAgentSessionAsync(testAgentHost); SetEnvironmentVariable("DD_ENV", string.Empty); SetEnvironmentVariable("DD_SERVICE", string.Empty); @@ -608,7 +395,7 @@ public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetric using (await RunSampleAndWaitForExit(agent, packageVersion: packageVersion ?? "1.13.1")) { - var metricsData = await WaitForTestAgentData($"http://{testAgentHost}:4318/test/session/metrics"); + var metricsData = await OtlpSnapshotHelper.WaitForTestAgentDataAsync($"http://{testAgentHost}:4318/test/session/metrics"); metricsData.Should().NotBeNullOrEmpty(); foreach (var attribute in metricsData.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.version')]")) @@ -654,7 +441,7 @@ public async Task SubmitsOtlpRuntimeMetrics() SkipOn.Platform(SkipOn.PlatformValue.MacOs); var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "localhost"; - await ClearTestAgentSession(testAgentHost); + await OtlpSnapshotHelper.ClearTestAgentSessionAsync(testAgentHost); SetEnvironmentVariable("DD_RUNTIME_METRICS_ENABLED", "true"); SetEnvironmentVariable("DD_METRICS_OTEL_ENABLED", "true"); @@ -667,7 +454,7 @@ public async Task SubmitsOtlpRuntimeMetrics() using var agent = EnvironmentHelper.GetMockAgent(useStatsD: true); using (await RunSampleAndWaitForExit(agent)) { - var metricsData = await WaitForTestAgentData($"http://{testAgentHost}:4318/test/session/metrics"); + var metricsData = await OtlpSnapshotHelper.WaitForTestAgentDataAsync($"http://{testAgentHost}:4318/test/session/metrics"); metricsData.Should().NotBeNullOrEmpty(); // Deduplicate metrics across multiple export intervals, keeping one per metric name @@ -741,7 +528,7 @@ public async Task SubmitsOtlpLogs(string packageVersion, string datadogLogsEnabl var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "localhost"; var otlpPort = protocol == "grpc" ? 4317 : 4318; - await ClearTestAgentSession(testAgentHost); + await OtlpSnapshotHelper.ClearTestAgentSessionAsync(testAgentHost); SetEnvironmentVariable("DD_ENV", "testing"); SetEnvironmentVariable("DD_SERVICE", "OtlpLogsService"); @@ -778,7 +565,7 @@ public async Task SubmitsOtlpLogs(string packageVersion, string datadogLogsEnabl { var endTimeNanoseconds = DateTimeOffset.UtcNow.ToUnixTimeNanoseconds(); - var logsData = await WaitForTestAgentData($"http://{testAgentHost}:4318/test/session/logs"); + var logsData = await OtlpSnapshotHelper.WaitForTestAgentDataAsync($"http://{testAgentHost}:4318/test/session/logs"); logsData.Should().NotBeNullOrEmpty(); logsData.SelectTokens("$..log_records[*]").Should().AllSatisfy(logRecord => { @@ -885,69 +672,6 @@ await Verifier.Verify(formattedJson, settings) } #endif - /// - /// Clears the test-agent session, retrying if the agent is not yet ready. - /// Ensures the OTLP HTTP endpoint is accepting connections before tests proceed. - /// - private static async Task ClearTestAgentSession(string testAgentHost, int maxRetries = 5, int delayMs = 1000) - { - using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(5) }; - var url = $"http://{testAgentHost}:4318/test/session/clear"; - - for (var attempt = 1; attempt <= maxRetries; attempt++) - { - try - { - var response = await httpClient.GetAsync(url); - response.EnsureSuccessStatusCode(); - return; - } - catch (Exception) when (attempt < maxRetries) - { - await Task.Delay(delayMs); - } - } - - // Final attempt -- let it throw if it fails - var finalResponse = await httpClient.GetAsync(url); - finalResponse.EnsureSuccessStatusCode(); - } - - /// - /// Polls the test-agent for data until non-empty results are returned or timeout is reached. - /// The sample app exports data during shutdown, so there can be a brief delay - /// between process exit and data appearing in the test-agent. The timeout is generous - /// because first-time gRPC connections (TCP+HTTP/2+TLS handshake) plus tracer shutdown - /// flushing can stack up on slower CI runners. - /// - private static async Task WaitForTestAgentData(string url, int timeoutSeconds = 60, int pollIntervalMs = 500) - { - using var httpClient = new HttpClient { Timeout = TimeSpan.FromSeconds(10) }; - var deadline = DateTime.UtcNow.AddSeconds(timeoutSeconds); - - while (DateTime.UtcNow < deadline) - { - var response = await httpClient.GetAsync(url); - response.EnsureSuccessStatusCode(); - - var json = await response.Content.ReadAsStringAsync(); - var data = JToken.Parse(json); - - if (data.HasValues) - { - return data; - } - - await Task.Delay(pollIntervalMs); - } - - // Final attempt -- return whatever we get so the caller's assertion shows the actual value - var finalResponse = await httpClient.GetAsync(url); - finalResponse.EnsureSuccessStatusCode(); - var finalJson = await finalResponse.Content.ReadAsStringAsync(); - return JToken.Parse(finalJson); - } - private static string GetSuffix(string packageVersion) { // The snapshots are only different in .NET Core 2.1 - .NET 5 with package version 1.0.1 @@ -975,18 +699,5 @@ private static string GetSuffix(string packageVersion) return string.Empty; } - - private static void AddProtobufToJsonScrubbers(VerifyTests.VerifySettings settings) - { - foreach (var (from, to) in ProtobufToJsonFieldNameMappings) - { - settings.AddSimpleScrubber(from, to); - } - - foreach (var (from, to) in ProtobufToJsonEnumMappings) - { - settings.AddSimpleScrubber(from, to); - } - } } }