From a26be874eb6d98615be5fab0ba465bb364ac233d Mon Sep 17 00:00:00 2001 From: Ishara Shanmugasundaram Date: Fri, 24 Jul 2026 13:58:47 -0400 Subject: [PATCH 1/3] Bridge Azure Durable Functions host OTel activities to Datadog spans Subscribe to the WebJobs.Extensions.DurableTask ActivitySource emitted by the Functions host when Distributed Tracing V2 is enabled, and map host-side orchestration and related durable task spans into Datadog with serverless semantics. Complements worker-side trace context propagation in dd-trace-js. Co-authored-by: Cursor --- .../Handlers/ActivityHandlersRegister.cs | 3 + .../Handlers/DurableTaskActivityHandler.cs | 64 +++++++++++++ .../DurableTaskActivityHandlerCommon.cs | 93 +++++++++++++++++++ .../Activity/Handlers/DurableTaskConstants.cs | 40 ++++++++ .../DurableTaskActivityHandlerCommonTests.cs | 57 ++++++++++++ 5 files changed, 257 insertions(+) create mode 100644 tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs create mode 100644 tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs create mode 100644 tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs create mode 100644 tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/ActivityHandlersRegister.cs b/tracer/src/Datadog.Trace/Activity/Handlers/ActivityHandlersRegister.cs index 2d9245a115cf..995fd47e4d5a 100644 --- a/tracer/src/Datadog.Trace/Activity/Handlers/ActivityHandlersRegister.cs +++ b/tracer/src/Datadog.Trace/Activity/Handlers/ActivityHandlersRegister.cs @@ -28,6 +28,9 @@ internal static class ActivityHandlersRegister // Quartz handlers new QuartzActivityHandler(), + // Azure Functions Durable Task host-side distributed tracing (V2) + new DurableTaskActivityHandler(), + // The default handler catches an activity and creates a datadog span from it. new DefaultActivityHandler(), }; diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs new file mode 100644 index 000000000000..c7c9bdeb0f32 --- /dev/null +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs @@ -0,0 +1,64 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using Datadog.Trace.Activity.DuckTypes; +using Datadog.Trace.Logging; +using Datadog.Trace.Tagging; + +namespace Datadog.Trace.Activity.Handlers +{ + /// + /// Bridges host-side Durable Functions distributed tracing activities emitted by the + /// WebJobs.Extensions.DurableTask ActivitySource into Datadog spans. + /// + internal sealed class DurableTaskActivityHandler : IActivityHandler + { + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); + + public bool ShouldListenTo(string sourceName, string? version) + => sourceName == DurableTaskConstants.ActivitySourceName; + + public void ActivityStarted(string sourceName, T activity) + where T : IActivity + { + var tags = new OpenTelemetryTags + { + OtelLibraryName = DurableTaskConstants.ActivitySourceName, + }; + + ActivityHandlerCommon.ActivityStarted(sourceName, activity, tags: tags, out _); + } + + public void ActivityStopped(string sourceName, T activity) + where T : IActivity + { + ActivityKey key = activity switch + { + IW3CActivity { TraceId: not null, SpanId: not null } w3cActivity => new(w3cActivity.TraceId, w3cActivity.SpanId), + _ => new(activity.Id) + }; + + if (key.IsValid() + && ActivityHandlerCommon.ActivityMappingById.TryRemove(key, out var activityMapping) + && activityMapping.Scope.Span is Span span) + { + DurableTaskActivityHandlerCommon.EnhanceSpan(span, activity); + OtlpHelpers.UpdateSpanFromActivity(activity, span, Tracer.Instance.Settings.OtelSemanticsEnabled); + span.Finish(activity.StartTimeUtc.Add(activity.Duration)); + activityMapping.Scope.Close(); + return; + } + + Log.Debug( + "Could not find span for Durable Task activity '{ActivityId}' with key '{Key}', falling back to common handler", + activity.Id, + key); + + ActivityHandlerCommon.ActivityStopped(sourceName, activity); + } + } +} diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs new file mode 100644 index 000000000000..17eaddc83ea7 --- /dev/null +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs @@ -0,0 +1,93 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Collections.Generic; +using Datadog.Trace.Activity.DuckTypes; +using Datadog.Trace.Tagging; + +namespace Datadog.Trace.Activity.Handlers +{ + internal static class DurableTaskActivityHandlerCommon + { + internal static string GetResourceName(string? operationName, string? taskName, string? taskType) + { + if (!string.IsNullOrEmpty(taskName)) + { + return taskType switch + { + DurableTaskConstants.TaskTypes.Orchestration => $"orchestration:{taskName}", + DurableTaskConstants.TaskTypes.CreateOrchestration => $"create_orchestration:{taskName}", + DurableTaskConstants.TaskTypes.Activity => $"activity:{taskName}", + DurableTaskConstants.TaskTypes.Entity => $"entity:{taskName}", + _ => taskName, + }; + } + + return operationName ?? "durabletask"; + } + + internal static string? MapActivityKindToSpanKind(ActivityKind activityKind) + => activityKind switch + { + ActivityKind.Server => SpanKinds.Server, + ActivityKind.Client => SpanKinds.Client, + ActivityKind.Producer => SpanKinds.Producer, + ActivityKind.Consumer => SpanKinds.Consumer, + _ => SpanKinds.Internal, + }; + + internal static bool TryGetTag(IEnumerable> tags, string key, out string? value) + { + foreach (var tag in tags) + { + if (tag.Key == key) + { + value = tag.Value?.ToString(); + return true; + } + } + + value = null; + return false; + } + + internal static void EnhanceSpan(Span span, T activity) + where T : IActivity + { + span.Type = SpanTypes.Serverless; + span.SetTag(Tags.InstrumentationName, nameof(Configuration.IntegrationId.AzureFunctions)); + + string? taskType = null; + string? taskName = null; + + if (activity is IActivity5 activity5) + { + var tags = activity5.TagObjects; + if (tags is not null) + { + TryGetTag(tags, DurableTaskConstants.Tags.Type, out taskType); + TryGetTag(tags, DurableTaskConstants.Tags.Name, out taskName); + + foreach (var tag in tags) + { + if (tag.Key.StartsWith("durabletask.", System.StringComparison.Ordinal)) + { + span.SetTag(tag.Key, tag.Value?.ToString()); + } + } + } + + if (span.Tags is OpenTelemetryTags otelTags) + { + otelTags.SpanKind = MapActivityKindToSpanKind(activity5.Kind); + } + } + + span.ResourceName = GetResourceName(activity.OperationName, taskName, taskType); + } + } +} diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs new file mode 100644 index 000000000000..83d6583f6f7f --- /dev/null +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs @@ -0,0 +1,40 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.Activity.Handlers +{ + /// + /// Constants for Azure Functions Durable Task host-side distributed tracing. + /// Mirrors Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation.Schema. + /// + internal static class DurableTaskConstants + { + internal const string ActivitySourceName = "WebJobs.Extensions.DurableTask"; + + internal static class Tags + { + internal const string Type = "durabletask.type"; + internal const string Name = "durabletask.task.name"; + internal const string Version = "durabletask.task.version"; + internal const string InstanceId = "durabletask.task.instance_id"; + internal const string ExecutionId = "durabletask.task.execution_id"; + internal const string TaskId = "durabletask.task.task_id"; + internal const string Operation = "durabletask.task.operation"; + } + + internal static class TaskTypes + { + internal const string Orchestration = "orchestration"; + internal const string CreateOrchestration = "create_orchestration"; + internal const string Activity = "activity"; + internal const string Entity = "entity"; + internal const string Event = "event"; + internal const string Timer = "timer"; + internal const string Client = "client"; + } + } +} diff --git a/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs b/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs new file mode 100644 index 000000000000..49ed6d009e66 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs @@ -0,0 +1,57 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System.Collections.Generic; +using Datadog.Trace.Activity.DuckTypes; +using Datadog.Trace.Activity.Handlers; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.Activity; + +public class DurableTaskActivityHandlerCommonTests +{ + [Theory] + [InlineData("orchestration", "MyOrchestrator", "orchestration:MyOrchestrator")] + [InlineData("create_orchestration", "MyOrchestrator", "create_orchestration:MyOrchestrator")] + [InlineData("activity", "MyActivity", "activity:MyActivity")] + [InlineData("entity", "MyEntity", "entity:MyEntity")] + [InlineData("timer", "MyOrchestrator", "MyOrchestrator")] + [InlineData(null, null, "create_orchestration:MyOrchestrator@1")] + public void GetResourceName_ReturnsExpectedValue(string? taskType, string? taskName, string expected) + { + var result = DurableTaskActivityHandlerCommon.GetResourceName( + operationName: "create_orchestration:MyOrchestrator@1", + taskName: taskName, + taskType: taskType); + + result.Should().Be(expected); + } + + [Fact] + public void TryGetTag_FindsMatchingTag() + { + var tags = new List> + { + new(DurableTaskConstants.Tags.Type, "orchestration"), + new(DurableTaskConstants.Tags.Name, "MyOrchestrator"), + }; + + DurableTaskActivityHandlerCommon.TryGetTag(tags, DurableTaskConstants.Tags.Name, out var value) + .Should().BeTrue(); + value.Should().Be("MyOrchestrator"); + } + + [Theory] + [InlineData(ActivityKind.Server, SpanKinds.Server)] + [InlineData(ActivityKind.Client, SpanKinds.Client)] + [InlineData(ActivityKind.Producer, SpanKinds.Producer)] + [InlineData(ActivityKind.Consumer, SpanKinds.Consumer)] + [InlineData(ActivityKind.Internal, SpanKinds.Internal)] + public void MapActivityKindToSpanKind_ReturnsExpectedValue(ActivityKind activityKind, string expected) + { + DurableTaskActivityHandlerCommon.MapActivityKindToSpanKind(activityKind).Should().Be(expected); + } +} From 3fb96d76d8afe2a0e4fc211d12b9ab8b925d8c7a Mon Sep 17 00:00:00 2001 From: Ishara Shanmugasundaram Date: Fri, 24 Jul 2026 14:21:46 -0400 Subject: [PATCH 2/3] Fix nullable build errors in DurableTask resource name helper Co-authored-by: Cursor --- .../Activity/Handlers/DurableTaskActivityHandlerCommon.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs index 17eaddc83ea7..07a03304b6b4 100644 --- a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandlerCommon.cs @@ -23,7 +23,7 @@ internal static string GetResourceName(string? operationName, string? taskName, DurableTaskConstants.TaskTypes.CreateOrchestration => $"create_orchestration:{taskName}", DurableTaskConstants.TaskTypes.Activity => $"activity:{taskName}", DurableTaskConstants.TaskTypes.Entity => $"entity:{taskName}", - _ => taskName, + _ => taskName!, }; } From 6f8e2d3f215e4b7eb63b495504043b7ee2d27bc3 Mon Sep 17 00:00:00 2001 From: Ishara Shanmugasundaram Date: Fri, 24 Jul 2026 14:29:15 -0400 Subject: [PATCH 3/3] Listen to Microsoft.DurableTask ActivitySource for orchestration spans V2 durable tracing emits orchestration execution spans from the Microsoft.DurableTask source, not only WebJobs.Extensions.DurableTask. Co-authored-by: Cursor --- .../Activity/Handlers/DurableTaskActivityHandler.cs | 5 +++-- .../Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs | 6 +++++- .../Activity/DurableTaskActivityHandlerCommonTests.cs | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs index c7c9bdeb0f32..0492b66b29d8 100644 --- a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskActivityHandler.cs @@ -20,14 +20,15 @@ internal sealed class DurableTaskActivityHandler : IActivityHandler private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(); public bool ShouldListenTo(string sourceName, string? version) - => sourceName == DurableTaskConstants.ActivitySourceName; + => sourceName == DurableTaskConstants.WebJobsActivitySourceName + || sourceName == DurableTaskConstants.SdkActivitySourceName; public void ActivityStarted(string sourceName, T activity) where T : IActivity { var tags = new OpenTelemetryTags { - OtelLibraryName = DurableTaskConstants.ActivitySourceName, + OtelLibraryName = sourceName, }; ActivityHandlerCommon.ActivityStarted(sourceName, activity, tags: tags, out _); diff --git a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs index 83d6583f6f7f..cf02edc1b8e6 100644 --- a/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs +++ b/tracer/src/Datadog.Trace/Activity/Handlers/DurableTaskConstants.cs @@ -13,7 +13,11 @@ namespace Datadog.Trace.Activity.Handlers /// internal static class DurableTaskConstants { - internal const string ActivitySourceName = "WebJobs.Extensions.DurableTask"; + // Host extension scheduling/metadata spans (create_orchestration, entity, etc.) + internal const string WebJobsActivitySourceName = "WebJobs.Extensions.DurableTask"; + + // Durable Task SDK execution spans (orchestration/activity/timer lifecycle in V2 tracing) + internal const string SdkActivitySourceName = "Microsoft.DurableTask"; internal static class Tags { diff --git a/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs b/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs index 49ed6d009e66..a990fbc62bf2 100644 --- a/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs +++ b/tracer/test/Datadog.Trace.Tests/Activity/DurableTaskActivityHandlerCommonTests.cs @@ -39,7 +39,7 @@ public void TryGetTag_FindsMatchingTag() new(DurableTaskConstants.Tags.Name, "MyOrchestrator"), }; - DurableTaskActivityHandlerCommon.TryGetTag(tags, DurableTaskConstants.Tags.Name, out var value) + DurableTaskActivityHandlerCommon.TryGetTag(tags, DurableTaskConstants.Tags.Name, out var value!) .Should().BeTrue(); value.Should().Be("MyOrchestrator"); }