From f6f457294b9933dee8869b8ea89d5fdc94b18f5d Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 10 Oct 2025 15:52:31 -0400 Subject: [PATCH 01/42] add temporary investigation files --- .../trace_payload_1762897459231.json | 1 + .../trace_payload_1762897460146.json | 1 + .../trace_payload_1762824450957.json | 1 + .../trace_payload_1762824451432.json | 1 + ...-981_ab644f9bfc6f421a8b58d26611dbfdde.json | 1 + ...-203_e4e0746e72de473dac6050a278cb36fd.json | 1 + ...MSVLS-58-Azure-Functions-span-parenting.md | 395 ++++++++++++++++++ 7 files changed, 401 insertions(+) create mode 100644 docs/development/investigations/1-original/trace_payload_1762897459231.json create mode 100644 docs/development/investigations/1-original/trace_payload_1762897460146.json create mode 100644 docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json create mode 100644 docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json create mode 100644 docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json create mode 100644 docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json create mode 100644 docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md diff --git a/docs/development/investigations/1-original/trace_payload_1762897459231.json b/docs/development/investigations/1-original/trace_payload_1762897459231.json new file mode 100644 index 000000000000..9657a3265ebe --- /dev/null +++ b/docs/development/investigations/1-original/trace_payload_1762897459231.json @@ -0,0 +1 @@ +[[{"trace_id":1910270346618876437,"span_id":4052593309550270339,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762897459030835400,"duration":115514400,"parent_id":13925130946451077401,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6913ae3200000000","runtime-id":"3e375b7a-5170-460c-bd77-278b812ca542","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":1910270346618876437,"span_id":13925130946451077401,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762897459015207300,"duration":135993900,"parent_id":18286165385944622934,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":1910270346618876437,"span_id":18286165385944622934,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762897458993111500,"duration":159499100,"parent_id":1978509666546725896,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"3e375b7a-5170-460c-bd77-278b812ca542","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/1-original/trace_payload_1762897460146.json b/docs/development/investigations/1-original/trace_payload_1762897460146.json new file mode 100644 index 000000000000..56f0aac1c7f6 --- /dev/null +++ b/docs/development/investigations/1-original/trace_payload_1762897460146.json @@ -0,0 +1 @@ +[[{"trace_id":1910270346618876437,"span_id":5775791465145012166,"name":"http.request","resource":"GET localhost:43239/api/HttpTest","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762897458789654200,"duration":413406600,"parent_id":1978509666546725896,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"http://localhost:43239/api/HttpTest","http-client-handler-type":"System.Net.Http.SocketsHttpHandler","http.status_code":"200","out.host":"localhost","_dd.p.dm":"-1","_dd.p.tid":"6913ae3200000000","runtime-id":"1ecec7fe-9bf8-4cd7-8a14-f47f80add4a8","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"b358dabba967e43e3c9b5e85bd392313864792a2","_dd.git.repository_url":"https://github.com/Azure/azure-functions-host","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"_dd.top_level":1}},{"trace_id":1910270346618876437,"span_id":1978509666546725896,"name":"azure_functions.invoke","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762897458672838500,"duration":635318800,"meta":{"component":"AzureFunctions","span.kind":"server","http.useragent":"curl/8.16.0","http.method":"GET","http.request.headers.host":"lucasp-premium-linux-isolated-aspnet.azurewebsites.net","http.url":"https://lucasp-premium-linux-isolated-aspnet.azurewebsites.net/api/HttpTest","http.status_code":"200","aas.function.name":"HttpTest","aas.function.binding":"Microsoft.Azure.WebJobs.Host.Executors.BindingSource","aas.function.trigger":"Http","runtime-id":"1ecec7fe-9bf8-4cd7-8a14-f47f80add4a8","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"process_id":27,"_dd.agent_psr":1,"_dd.tracer_kr":1,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json new file mode 100644 index 000000000000..7e035b1a6be1 --- /dev/null +++ b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json @@ -0,0 +1 @@ +[[{"trace_id":11227614026327825028,"span_id":3342051758595645288,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762824450609760600,"duration":170438800,"parent_id":9984891158266111958,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":11227614026327825028,"span_id":9984891158266111958,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762824450594507000,"duration":190324400,"parent_id":727838728754785615,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":11227614026327825028,"span_id":727838728754785615,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762824450568570100,"duration":217483300,"parent_id":12539548146409755932,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}],[{"trace_id":11227614026327825028,"span_id":5312880851904873807,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762824450422449400,"duration":420572900,"parent_id":17651134856053342758,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"localhost:43229","http.url":"http://localhost:43229/api/HttpTest","http.status_code":"200","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json new file mode 100644 index 000000000000..b32d179a2e56 --- /dev/null +++ b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json @@ -0,0 +1 @@ +[[{"trace_id":11227614026327825028,"span_id":17651134856053342758,"name":"http.request","resource":"GET localhost:43229/api/HttpTest","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762824450339164400,"duration":497064700,"parent_id":12539548146409755932,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"http://localhost:43229/api/HttpTest","http-client-handler-type":"System.Net.Http.SocketsHttpHandler","http.status_code":"200","out.host":"localhost","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"5210fd98-a24d-4204-9f43-118a5fa2c857","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"b358dabba967e43e3c9b5e85bd392313864792a2","_dd.git.repository_url":"https://github.com/Azure/azure-functions-host","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"_dd.top_level":1}},{"trace_id":11227614026327825028,"span_id":12539548146409755932,"name":"azure_functions.invoke","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762824450215590500,"duration":784896500,"meta":{"component":"AzureFunctions","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"lucasp-premium-linux-isolated-aspnet.azurewebsites.net","http.url":"https://lucasp-premium-linux-isolated-aspnet.azurewebsites.net/api/HttpTest","http.status_code":"200","aas.function.name":"HttpTest","aas.function.binding":"Microsoft.Azure.WebJobs.Host.Executors.BindingSource","aas.function.trigger":"Http","runtime-id":"5210fd98-a24d-4204-9f43-118a5fa2c857","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"process_id":27,"_dd.agent_psr":1,"_dd.tracer_kr":1,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json new file mode 100644 index 000000000000..c3be8a9f6525 --- /dev/null +++ b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json @@ -0,0 +1 @@ +[[{"trace_id":17961676107601563275,"span_id":4768510696171411833,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762980468406397700,"duration":473585100,"parent_id":7380191810641356982,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6914f27400000000","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":17961676107601563275,"span_id":7380191810641356982,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762980468391630100,"duration":492717200,"parent_id":15573027893036604679,"meta":{"process.id":"59","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":17961676107601563275,"span_id":15573027893036604679,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762980468368988400,"duration":516924400,"parent_id":4682913969503290245,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":59,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}],[{"trace_id":17961676107601563275,"span_id":4316486117327122103,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762980468236513800,"duration":704521600,"parent_id":243437182453255327,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"localhost:42959","http.url":"http://localhost:42959/api/HttpTest","http.status_code":"200","_dd.p.dm":"-1","_dd.p.tid":"6914f27400000000","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":59,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json new file mode 100644 index 000000000000..e8f580d623ed --- /dev/null +++ b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json @@ -0,0 +1 @@ +[[{"trace_id":14656220060439490006,"span_id":3397667712105060208,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762987094553261900,"duration":148844700,"parent_id":11352266471047046875,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"69150c5500000000","runtime-id":"99f59032-46b6-4cf8-8525-0a6d3e7d33f1","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":14656220060439490006,"span_id":11352266471047046875,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762987094530674000,"duration":179029900,"parent_id":1243553223469235235,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":14656220060439490006,"span_id":1243553223469235235,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762987094493292400,"duration":217723200,"parent_id":10174520177415259312,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":14656220060439490006,"span_id":10174520177415259312,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762987094280273800,"duration":474975900,"parent_id":17403981112360440532,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"curl/8.16.0","http.method":"GET","http.request.headers.host":"localhost:40813","http.url":"http://localhost:40813/api/HttpTest","http.status_code":"200","runtime-id":"99f59032-46b6-4cf8-8525-0a6d3e7d33f1","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md b/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md new file mode 100644 index 000000000000..5cfa90edd61b --- /dev/null +++ b/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md @@ -0,0 +1,395 @@ +# APMSVLS-58: Azure Functions Span Parenting Issue + +**Status**: ✅ **RESOLVED** (Phase 3 Complete) +**Branch**: `lpimentel/APMSVLS-58-azfunc-host-parenting` +**Date Started**: October 2025 +**Date Resolved**: November 12, 2025 + +## Related Documentation + +- [AzureFunctions.md](../AzureFunctions.md) - Azure Functions integration guide +- [AzureFunctions-Architecture.md](../AzureFunctions-Architecture.md) - Architecture deep dive +- [QueryingDatadogAPIs.md](../QueryingDatadogAPIs.md) - Query Datadog APIs for debugging + +## Problem Description + +In isolated Azure Functions with ASP.NET Core Integration, the worker's `azure_functions.invoke` span is incorrectly parented, resulting in broken distributed traces. + +### Current (Incorrect) Behavior + +**After enabling AspNetCoreDiagnosticObserver in worker** (Phase 2): + +``` +ROOT: azure_functions.invoke [HOST] + └─ http.request [HOST → WORKER] + └─ aspnet_core.request [WORKER] ✓ Correct (now exists!) + +Worker azure_functions.invoke in same trace but WRONG parent: +└─ azure_functions.invoke [WORKER] + └─ parent: host's root span ❌ Should be child of aspnet_core.request + ├─ test_span [WORKER] + └─ http.request [WORKER] +``` + +### Desired Behavior + +``` +ROOT: aspnet_core.request [WORKER] + └─ azure_functions.invoke [WORKER] ✓ Correct + ├─ test_span [WORKER] + └─ http.request [WORKER] + +Note: Host-side azure_functions.invoke and http.request spans should be removed +``` + +**Key observations**: +- All spans tagged with `aas.function.process:host` or `aas.function.process:worker` +- Worker's `azure_functions.invoke` is parented to **host's root span** instead of `aspnet_core.request` +- Need to: (1) Fix worker span parenting (high priority), (2) Remove host-side spans + +## Root Cause (RESOLVED) + +**Primary Issue**: Looking for wrong key in `FunctionContext.Items` +- ❌ Was using: `"__AspNetCoreHttpContext__"` (doesn't exist) +- ✅ Should use: `"HttpRequestContext"` (actual key set by `FunctionsHttpProxyingMiddleware`) + +**Secondary Issue**: Stale header extraction +- In ASP.NET Core mode, headers in gRPC message contain host's root span context +- These stale headers were used as fallback when HttpContext.Items lookup failed +- Worker's `azure_functions.invoke` span incorrectly parented to host's root span + +**Underlying Cause**: AsyncLocal context doesn't flow through Azure Functions middleware, so HttpContext.Items bridge is required. + +## Investigation Findings + +### Confirmed Working ✅ + +1. HTTP proxying: Host returns empty gRPC message when `isHttpProxying` is true +2. HTTP client instrumentation creates `http.request` span and injects trace context headers +3. Worker's `aspnet_core.request` span correctly parented to host's `http.request` span + +### Current Issues ❌ + +1. **AsyncLocal doesn't flow**: `tracer.InternalActiveScope` is null when creating `azure_functions.invoke` span +2. **Activity.Current is broken in Azure Functions**: Cannot rely on `System.Diagnostics.Activity.Current` +3. **Incorrect parent extraction**: Worker's span gets host's root span context instead of `aspnet_core.request` context + +## Trace Analysis + +### Phase 1: Original Behavior (Before AspNetCoreDiagnosticObserver Enabled) + +**Location**: [`1-original/`](1-original/) + +Before enabling AspNetCoreDiagnosticObserver in the worker process, the `aspnet_core.request` span was **not created at all** because the observer was disabled in all Azure Functions processes (both host and worker). + +**Captured traces**: +- [trace_payload_1762897459231.json](1-original/trace_payload_1762897459231.json) - Worker spans +- [trace_payload_1762897460146.json](1-original/trace_payload_1762897460146.json) - Host spans + +**Analysis of trace `1910270346618876437`**: +``` +HOST PROCESS (runtime_id: 1ecec7fe, process: 27): +├─ azure_functions.invoke (span: 1978509666546725896) [ROOT] ❌ +│ resource: "GET /api/httptest" +│ duration: 635ms +│ +└─ http.request (span: 5775791465145012166) ❌ + resource: "GET localhost:43239/api/HttpTest" + duration: 413ms + parent: 1978509666546725896 (host's azure_functions.invoke) + +WORKER PROCESS (runtime_id: 3e375b7a, process: 58): +└─ azure_functions.invoke (span: 18286165385944622934) ⚠️ + resource: "Http HttpTest" + duration: 159ms + parent: 1978509666546725896 (HOST's root span!) + │ + └─ test_span (span: 13925130946451077401) + duration: 136ms + │ + └─ http.request (span: 4052593309550270339) + resource: "GET jsonplaceholder.typicode.com/users/?" + duration: 115ms +``` + +**Key issues**: +- ❌ No `aspnet_core.request` span exists in worker process +- ❌ Worker's `azure_functions.invoke` is parented to HOST's root span (1978509666546725896) +- ❌ Host creates unnecessary `azure_functions.invoke` and `http.request` spans +- ✓ All spans are in the same trace (distributed tracing working via headers) + +### Phase 2: After Enabling AspNetCoreDiagnosticObserver in Worker + +**Location**: [`2-after-enabling-aspnetcore-observer-in-worker-process/`](2-after-enabling-aspnetcore-observer-in-worker-process/) + +After enabling AspNetCoreDiagnosticObserver in the worker process, the `aspnet_core.request` span is now created, but the worker's `azure_functions.invoke` span has incorrect parenting. + +**Captured traces**: +- [trace_payload_1762824450957.json](2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json) - Worker spans +- [trace_payload_1762824451432.json](2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json) - Host spans + +**Analysis of trace `11227614026327825028`**: +``` +Host spans: +├─ azure_functions.invoke (span: 12539548146409755932) [ROOT] ❌ Should not exist +└─ http.request (span: 17651134856053342758) [child of above] ❌ Should not exist + +Worker spans (all in same trace ✓): +├─ aspnet_core.request (span: 5312880851904873807) ✅ NOW EXISTS! +│ └─ parent: 17651134856053342758 (host's http.request) ✓ Correct! +└─ azure_functions.invoke (span: 727838728754785615) + └─ parent: 12539548146409755932 (host's root) ❌ Should be child of aspnet_core.request! +``` + +**Progress made**: +- ✅ `aspnet_core.request` span now exists in worker process +- ✅ `aspnet_core.request` correctly parented to host's `http.request` span +- ❌ Worker's `azure_functions.invoke` still parented to **host's root span** instead of `aspnet_core.request` +- ❌ Host still creates unnecessary `azure_functions.invoke` and `http.request` spans + +### Phase 3: Fixed HttpContext.Items Bridge ✅ RESOLVED + +**Commit**: `1d3179aa5` - Fix Azure Functions span parenting with ASP.NET Core + +**Root cause identified**: Looking for wrong key in `FunctionContext.Items` +- ❌ Old: `"__AspNetCoreHttpContext__"` (doesn't exist) +- ✅ Fixed: `"HttpRequestContext"` (actual key set by Azure Functions Worker SDK) + +**Changes made**: +1. Skip stale gRPC header extraction when ASP.NET Core integration detected (`AzureFunctionsCommon.cs:242-243`) +2. Use correct `"HttpRequestContext"` key for HttpContext lookup (`AzureFunctionsCommon.cs:292`) +3. Added comprehensive debug logging for troubleshooting + +**Analysis of trace `14656220060439490006`** (Nov 12, 2025): +``` +Worker spans: +├─ aspnet_core.request (span: 10174520177415259312) +│ └─ azure_functions.invoke (span: 1243553223469235235) ✅ CORRECT PARENT! +│ └─ test_span (span: 11352266471047046875) +│ └─ http.request (span: 3397667712105060208) +``` + +**Result**: +- ✅ Worker's `azure_functions.invoke` correctly parented to `aspnet_core.request` +- ✅ Proper span hierarchy in worker process +- ✅ All spans in same trace +- ⚠️ Host spans still present (secondary priority) + +## Code Changes Made + +### 1. Enable AspNetCoreDiagnosticObserver in Worker Process + +**File**: `tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs:473-493` + +Modified to enable `AspNetCoreDiagnosticObserver` in isolated worker process only: + +```csharp +var isInAzureFunctionsHost = EnvironmentHelpers.IsRunningInAzureFunctionsHost(); +var shouldSkipAspNetCore = isInAzureFunctionsHost || + (EnvironmentHelpers.IsAzureFunctions() && !EnvironmentHelpers.IsAzureFunctionsIsolated()); + +if (shouldSkipAspNetCore) +{ + Log.Debug("Skipping AspNetCoreDiagnosticObserver in Azure Functions (host or in-process)."); +} +else +{ + observers.Add(new AspNetCoreDiagnosticObserver()); +} +``` + +### 2. Add Process Identification Tag + +**File**: `tracer/src/Datadog.Trace/Agent/MessagePack/SpanMessagePackFormatter.cs:742-754` + +Added `aas.function.process` tag to distinguish host vs worker spans: +- Host spans: `aas.function.process: host` +- Worker spans: `aas.function.process: worker` + +## Testing Workflow + +### Environment + +**All Function Apps** (Resource Group: `lucas.pimentel`, Location: Canada Central): + +| Name | Purpose | +|------|---------| +| lucasp-premium-linux-isolated-aspnet | **Primary test app** - Isolated .NET 8 with ASP.NET Core Integration | +| lucasp-premium-linux-isolated | Isolated .NET 8 (no ASP.NET Core) | +| lucasp-premium-linux-inproc | In-process .NET 6 | +| lucasp-premium-windows-isolated-aspnet | Windows isolated with ASP.NET Core | +| lucasp-premium-windows-isolated | Windows isolated (no ASP.NET Core) | +| lucasp-premium-windows-inproc | Windows in-process | +| lucasp-consumption-windows-isolated | Windows consumption plan | +| lucasp-flex-consumption-isolated | Flex consumption plan | + +**Primary Test App:** +- **Name**: `lucasp-premium-linux-isolated-aspnet` +- **Resource Group**: `lucas.pimentel` +- **Source**: `D:\source\datadog\serverless-dev-apps\azure\functions\dotnet\isolated-dotnet8-aspnetcore` + +### Steps + +1. **Build NuGet package**: + ```powershell + .\tracer\tools\Build-AzureFunctionsNuget.ps1 -CopyTo D:\temp\nuget -Verbose + ``` + +2. **Deploy test app**: + ```bash + cd D:/source/datadog/serverless-dev-apps/azure/functions/dotnet/isolated-dotnet8-aspnetcore + dotnet restore + func azure functionapp publish lucasp-premium-linux-isolated + ``` + + Wait 1-2 minutes for worker restart after deployment. + +3. **Trigger function**: + ```bash + curl https://lucasp-premium-linux-isolated.azurewebsites.net/api/HttpTest + ``` + +4. **Download logs**: + ```bash + az functionapp log download \ + --name lucasp-premium-linux-isolated \ + --resource-group lucas.pimentel \ + --log-path D:/temp/logs-$(date +%H%M%S).zip + ``` + +5. **Query Datadog** (optional): + ```bash + # Filter by process type + curl -X POST https://api.datadoghq.com/api/v2/spans/events/search \ + -H "DD-API-KEY: ${DD_API_KEY}" \ + -H "DD-APPLICATION-KEY: ${DD_APPLICATION_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"data":{"attributes":{"filter":{"query":"service:lucasp-premium-linux-isolated @aas.function.process:worker","from":"now-10m","to":"now"}},"type":"search_request"}}' + ``` + +### Verification ✅ + +**Expected trace structure** (after fix): +``` +Worker aspnet_core.request (ROOT) +└─ Worker azure_functions.invoke (child) ✓ + └─ test_span + └─ http.request +``` + +**Verified** (Trace ID: `14656220060439490006`, Nov 12 2025): +- ✅ Worker's `azure_functions.invoke` parent_id matches `aspnet_core.request` span_id +- ✅ All spans in same trace +- ✅ Proper span hierarchy in worker process +- ⚠️ Host spans still present (secondary priority to remove) + +## Minimal Reproduction + +```csharp +[Function(nameof(HttpTest))] +public async Task HttpTest([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequest request) +{ + using (var scope = Tracer.Instance.StartActive("test_span")) + { + using var httpClient = new HttpClient(); + await httpClient.GetStringAsync("https://jsonplaceholder.typicode.com/users/1"); + return new OkObjectResult(new { message = "success" }); + } +} +``` + +**Requirements**: +- Isolated Azure Functions (.NET Isolated) +- ASP.NET Core Integration (`ConfigureFunctionsWebApplication()`) +- HTTP Trigger +- `Datadog.AzureFunctions` NuGet package + +## Goal + +Now that we have an `aspnet_core.request` span in the worker process (Phase 2), we need to: + +1. **Make worker's `azure_functions.invoke` span a child of `aspnet_core.request`** (high priority) + - Currently: Worker's `azure_functions.invoke` is parented to host's root span + - Goal: Worker's `azure_functions.invoke` should be parented to worker's `aspnet_core.request` span + - This will create the correct hierarchy: `aspnet_core.request` → `azure_functions.invoke` → user code + +2. **Remove host-side spans** (secondary priority) + - Host's `azure_functions.invoke` and `http.request` spans are unnecessary when ASP.NET Core integration is enabled + - These spans represent HTTP proxying overhead, not the actual function execution + - Detect HTTP proxying in host process and skip span creation + +## Implementation Plan + +### Approach: Use HttpContext.Items as Bridge + +Since AsyncLocal context doesn't flow through Azure Functions middleware, use `HttpContext.Items` to explicitly pass span context between middleware layers. + +**Step 1: Store scope in HttpContext.Items** +- File: `tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs:125` +- After creating scope, store in `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` +- Use `__` prefix to avoid conflicts with user code (same pattern as `TracingHttpModule.cs`) + +**Step 2: Retrieve scope in Azure Functions middleware** +- File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs:262-279` +- Get HttpContext from `functionContext.GetHttpContext()` +- Retrieve scope from `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` +- Use as parent before falling back to extraction logic + +**Step 3: Skip host span creation when proxying** +- File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsExecutorTryExecuteAsyncIntegration.cs` +- Detect `IsRunningInAzureFunctionsHost()` + HTTP proxying enabled +- Skip span creation in host when ASP.NET Core integration is active in worker + +**Why this works:** +- HttpContext.Items persists throughout request lifecycle +- Both ASP.NET Core and Azure Functions middleware access same HttpContext instance +- No reliance on AsyncLocal or Activity.Current +- Explicit and debuggable + +**Fallback approaches:** +- Use `FunctionContext.Features` to store/retrieve scope +- Custom middleware to bridge AsyncLocal → Features before context is lost + +## Implementation Progress + +### Phase 3: HttpContext.Items Bridge (In Progress) + +**Status**: Code implemented, testing pending + +**Changes Made** (Commit: d0e31854a): + +1. **Store scope in HttpContext.Items** ✅ + - File: `tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs:142` + - Added: `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope;` + - Stores the AspNetCore scope immediately after creation + +2. **Add Items property to IFunctionContext** ✅ + - File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs:22` + - Added: `IDictionary? Items { get; }` + - Allows duck-typed access to FunctionContext.Items + +3. **Retrieve scope from HttpContext.Items** ✅ + - File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs:262-297` + - Logic: + 1. Check if `tracer.InternalActiveScope == null` (AsyncLocal didn't flow) + 2. Try to get HttpContext from `context.Items["__AspNetCoreHttpContext__"]` + 3. Try to get scope from `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` + 4. Use scope as parent if found, otherwise fall back to header extraction + - Only uses HttpContext.Items when AsyncLocal fails (maintains backward compatibility) + +**Next Steps**: +1. Deploy updated NuGet package to `lucasp-premium-linux-isolated-aspnet` +2. Trigger HttpTest function +3. Download and analyze traces to verify: + - Worker's `azure_functions.invoke` is now child of `aspnet_core.request` + - All spans in same trace + - Correct parent_id relationships + +**Pending**: +- Step 3: Skip host span creation when HTTP proxying is detected (secondary priority) + +## References + +- [Azure Functions ASP.NET Core Integration](https://learn.microsoft.com/en-us/azure/azure-functions/dotnet-isolated-process-guide?tabs=hostbuilder%2Cwindows#aspnet-core-integration) +- [Azure Functions Host source](https://github.com/Azure/azure-functions-host) +- [YARP (Yet Another Reverse Proxy)](https://microsoft.github.io/reverse-proxy/) From cfaaca787c9cbda804504fe5adc82c4e0e90e3ed Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 11 Nov 2025 18:45:16 -0500 Subject: [PATCH 02/42] Fix Azure Functions span parenting via HttpContext.Items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AsyncLocal context doesn't flow through Azure Functions middleware, causing worker's azure_functions.invoke span to be incorrectly parented. Use HttpContext.Items as an explicit bridge to pass the AspNetCore scope to the Azure Functions middleware. Changes: - Store AspNetCore scope in HttpContext.Items after creation - Add Items property to IFunctionContext duck type interface - Retrieve scope from HttpContext.Items when creating azure_functions.invoke span - Only use HttpContext.Items fallback when tracer.InternalActiveScope is null 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../Azure/Functions/AzureFunctionsCommon.cs | 72 +++++++++++++++++-- .../Functions/Isolated/IFunctionContext.cs | 2 + .../AspNetCoreHttpRequestHandler.cs | 5 ++ .../Functions/AzureFunctionsCommonTests.cs | 2 + 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 2718e159b641..1c9c1bfbf8aa 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -254,8 +254,27 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even switch (triggerType) { case "Http": - extractedContext = ExtractPropagatedContextFromHttp(functionContext, entry.Key as string).MergeBaggageInto(Baggage.Current); + { + // Detect ASP.NET Core integration by checking for HttpContext in FunctionContext.Items + // In ASP.NET Core mode, HTTP requests are proxied directly (not via gRPC) + // The headers in the gRPC message are STALE (contain host's root span context) + // The key "HttpRequestContext" is set by FunctionsHttpProxyingMiddleware in the worker + var isAspNetCoreIntegration = functionContext.Items?.ContainsKey("HttpRequestContext") == true; + + if (isAspNetCoreIntegration) + { + // Skip extraction - will rely on HttpContext.Items bridge or create root span + Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected (HTTP proxying mode)"); + } + else + { + // Only extract from gRPC message when NOT using ASP.NET Core integration + extractedContext = ExtractPropagatedContextFromHttp(functionContext, entry.Key as string).MergeBaggageInto(Baggage.Current); + Log.Debug("Extracted trace context from gRPC message (non-ASP.NET Core mode)"); + } + break; + } case "ServiceBus" when tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(IntegrationId.AzureServiceBus): extractedContext = ExtractPropagatedContextFromMessaging(functionContext, "UserProperties", "UserPropertiesArray").MergeBaggageInto(Baggage.Current); @@ -279,12 +298,57 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even ShortName = functionName, FullName = functionContext.FunctionDefinition.EntryPoint, }; - tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); - scope = tracer.StartActiveInternal(OperationName, tags: tags, parent: extractedContext.SpanContext); + + Log.Debug("Azure Functions span creation: AsyncLocal context not available - attempting HttpContext.Items bridge"); + + // AsyncLocal context didn't flow - try to get parent scope from HttpContext.Items + // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow + Scope? parentScope = null; + try + { + if (functionContext.Items != null && + functionContext.Items.TryGetValue("HttpRequestContext", out var httpContextObj) && + httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && + httpContext.Items.TryGetValue("__Datadog.Trace.AspNetCore.ActiveScope", out var scopeObj) && + scopeObj is Scope aspNetCoreScope) + { + parentScope = aspNetCoreScope; + Log.Debug("Azure Functions span creation: Retrieved AspNetCore scope - span_id: {SpanId}, trace_id: {TraceId}", aspNetCoreScope.Span.SpanId, aspNetCoreScope.Span.TraceId); + } + else + { + Log.Debug("Azure Functions span creation: Could not retrieve AspNetCore scope from HttpContext.Items"); + } + } + catch (Exception ex) + { + Log.Debug(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); + } + + if (parentScope != null) + { + // Use the AspNetCore scope as parent + scope = tracer.StartActiveInternal(OperationName, parent: parentScope.Span.Context, tags: tags); + Log.Debug("Azure Functions span creation: Parented to AspNetCore scope - parent_id: {ParentId}", parentScope.Span.SpanId); + } + else if (extractedContext.SpanContext != null) + { + // Use extracted context from headers + tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); + scope = tracer.StartActiveInternal(OperationName, tags: tags, parent: extractedContext.SpanContext); + Log.Debug("Azure Functions span creation: Parented to extracted context - parent_id: {ParentId}, trace_id: {TraceId}", extractedContext.SpanContext.SpanId, extractedContext.SpanContext.TraceId); + } + else + { + // No parent available - create root span + tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); + scope = tracer.StartActiveInternal(OperationName, tags: tags); + Log.Debug("Azure Functions span creation: Created as root span (no parent available)"); + } } else { - // shouldn't be hit, but better safe than sorry + // AsyncLocal is working - use it as parent scope = tracer.StartActiveInternal(OperationName); var rootSpan = scope.Root.Span; diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs index 6df59c1ebbc8..91ed96e88b56 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs @@ -18,6 +18,8 @@ internal interface IFunctionContext FunctionDefinitionStruct FunctionDefinition { get; } IEnumerable>? Features { get; } + + IDictionary? Items { get; } } #endif diff --git a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs index 5d5a56580988..17c227845357 100644 --- a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs +++ b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs @@ -174,6 +174,11 @@ private Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, Ias httpContext.Items[HttpContextTrackingKey] = new RequestTrackingFeature(originalPath, scope, proxyContext?.Scope); #endif + // Store scope in HttpContext.Items for Azure Functions middleware to retrieve + // Use __ prefix to avoid conflicts with user code (same pattern as TracingHttpModule) + httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope; + _log.Debug("AspNetCore: Stored scope in HttpContext.Items - span_id: {SpanId}, trace_id: {TraceId}, path: {Path}", scope.Span.SpanId, scope.Span.TraceId, request.Path); + if (tracer.Settings.IpHeaderEnabled || security.AppsecEnabled) { var peerIp = new Headers.Ip.IpInfo(httpContext.Connection.RemoteIpAddress?.ToString(), httpContext.Connection.RemotePort); diff --git a/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs b/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs index 0c39c2f92a1a..e60e6dd7879c 100644 --- a/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs +++ b/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs @@ -122,6 +122,8 @@ private class MockFunctionContext : IFunctionContext public FunctionDefinitionStruct FunctionDefinition { get; set; } public IEnumerable>? Features { get; set; } + + public IDictionary? Items { get; } } // This duck types with tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/GrpcBindingsFeatureStruct.cs From d144590678adfaa41b5e6fb7ca27ffaa7b1c89a6 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 13 Nov 2025 18:31:57 -0500 Subject: [PATCH 03/42] refactoring --- .../Azure/Functions/AzureFunctionsCommon.cs | 99 ++++++++++--------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 1c9c1bfbf8aa..6f02fa65a019 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -289,68 +289,39 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even } var functionName = functionContext.FunctionDefinition.Name; - if (tracer.InternalActiveScope == null) + + var tags = new AzureFunctionsTags { - // This is the root scope - var tags = new AzureFunctionsTags - { - TriggerType = triggerType, - ShortName = functionName, - FullName = functionContext.FunctionDefinition.EntryPoint, - }; + TriggerType = triggerType, + ShortName = functionName, + FullName = functionContext.FunctionDefinition.EntryPoint, + }; - Log.Debug("Azure Functions span creation: AsyncLocal context not available - attempting HttpContext.Items bridge"); + // If active scope didn't flow via AsyncLocal, try to get it from HttpContext.Items + // (for HTTP triggers using ASP.NET Core integration). + // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow. + var parentScope = tracer.InternalActiveScope ?? GetAspNetCoreScope(functionContext); - // AsyncLocal context didn't flow - try to get parent scope from HttpContext.Items - // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow - Scope? parentScope = null; - try - { - if (functionContext.Items != null && - functionContext.Items.TryGetValue("HttpRequestContext", out var httpContextObj) && - httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && - httpContext.Items.TryGetValue("__Datadog.Trace.AspNetCore.ActiveScope", out var scopeObj) && - scopeObj is Scope aspNetCoreScope) - { - parentScope = aspNetCoreScope; - Log.Debug("Azure Functions span creation: Retrieved AspNetCore scope - span_id: {SpanId}, trace_id: {TraceId}", aspNetCoreScope.Span.SpanId, aspNetCoreScope.Span.TraceId); - } - else - { - Log.Debug("Azure Functions span creation: Could not retrieve AspNetCore scope from HttpContext.Items"); - } - } - catch (Exception ex) - { - Log.Debug(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); - } + if (parentScope == null) + { + // no local parent available, we are creating a local root span + tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); + scope = tracer.StartActiveInternal(OperationName, parent: extractedContext.SpanContext, tags: tags); - if (parentScope != null) - { - // Use the AspNetCore scope as parent - scope = tracer.StartActiveInternal(OperationName, parent: parentScope.Span.Context, tags: tags); - Log.Debug("Azure Functions span creation: Parented to AspNetCore scope - parent_id: {ParentId}", parentScope.Span.SpanId); - } - else if (extractedContext.SpanContext != null) + if (extractedContext.SpanContext is { } extractedSpanContext) { - // Use extracted context from headers - tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); - scope = tracer.StartActiveInternal(OperationName, tags: tags, parent: extractedContext.SpanContext); - Log.Debug("Azure Functions span creation: Parented to extracted context - parent_id: {ParentId}, trace_id: {TraceId}", extractedContext.SpanContext.SpanId, extractedContext.SpanContext.TraceId); + Log.Debug("Azure Functions span creation: Parented to extracted context. parent_id: {ParentId}, trace_id: {TraceId}", extractedSpanContext.SpanId, extractedSpanContext.TraceId); } else { - // No parent available - create root span - tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); - scope = tracer.StartActiveInternal(OperationName, tags: tags); Log.Debug("Azure Functions span creation: Created as root span (no parent available)"); } } else { - // AsyncLocal is working - use it as parent - scope = tracer.StartActiveInternal(OperationName); + scope = tracer.StartActiveInternal(OperationName, parent: parentScope.Span.Context, tags: tags); + // copy some tags to the root span var rootSpan = scope.Root.Span; AzureFunctionsTags.SetRootSpanTags( rootSpan, @@ -375,6 +346,38 @@ httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && return scope; } + private static Scope? GetAspNetCoreScope(T functionContext) + where T : IFunctionContext + { + Log.Debug("Azure Functions span creation: AsyncLocal context not available - attempting HttpContext.Items bridge"); + + // AsyncLocal context didn't flow - try to get parent scope from HttpContext.Items + // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow + Scope? parentScope = null; + try + { + if (functionContext.Items != null && + functionContext.Items.TryGetValue("HttpRequestContext", out var httpContextObj) && + httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && + httpContext.Items.TryGetValue("__Datadog.Trace.AspNetCore.ActiveScope", out var scopeObj) && + scopeObj is Scope aspNetCoreScope) + { + parentScope = aspNetCoreScope; + Log.Debug("Azure Functions span creation: Retrieved AspNetCore scope - span_id: {SpanId}, trace_id: {TraceId}", aspNetCoreScope.Span.SpanId, aspNetCoreScope.Span.TraceId); + } + else + { + Log.Debug("Azure Functions span creation: Could not retrieve AspNetCore scope from HttpContext.Items"); + } + } + catch (Exception ex) + { + Log.Debug(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); + } + + return parentScope; + } + private static PropagationContext ExtractPropagatedContextFromHttp(T context, string? bindingName) where T : IFunctionContext { From 63839cb823289d1a591566b2677c90117a712a99 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Mon, 17 Nov 2025 18:43:12 -0500 Subject: [PATCH 04/42] update test snapshots --- ...AzureFunctionsTests.InProcess.verified.txt | 22 +- ...sTests.Isolated.V4.AspNetCore.verified.txt | 483 +++++++++++------- ...Tests.Isolated.V4.AspNetCore1.verified.txt | 483 +++++++++++------- 3 files changed, 621 insertions(+), 367 deletions(-) diff --git a/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt index be62a8395185..78b237eb8db5 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt @@ -13,9 +13,11 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Triggers.TriggerBindingSource`1[[Microsoft.Azure.WebJobs.TimerInfo, Microsoft.Azure.WebJobs.Extensions, Version=0.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35]], + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, aas.function.name: TriggerAllTimer, aas.function.trigger: Timer, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -75,13 +77,15 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, aas.function.name: TriggerCaller, aas.function.trigger: Http, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: aspnet_core, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, @@ -256,13 +260,15 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, aas.function.name: SimpleHttpTrigger, aas.function.trigger: Http, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: aspnet_core, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, @@ -295,13 +301,15 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, aas.function.name: Exception, aas.function.trigger: Http, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: aspnet_core, env: integration_tests, error.msg: Exception while executing function: Exception, error.stack: @@ -340,13 +348,15 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequest req, ILo aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, aas.function.name: ServerError, aas.function.trigger: Http, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: aspnet_core, env: integration_tests, error.msg: The HTTP response has status code 500., http.method: GET, @@ -379,13 +389,15 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequest req, ILo aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, aas.function.name: BadRequest, aas.function.trigger: Http, + aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: aspnet_core, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt index 2a6e19c64b29..554cde39b3e5 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt @@ -2,24 +2,102 @@ { TraceId: Id_1, SpanId: Id_2, + Name: aspnet_core.request, + Resource: GET /api/badrequest, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_3, + Tags: { + aas.environment.extension_version: unknown, + aas.environment.instance_id: unknown, + aas.environment.instance_name: IntegrationTestHost, + aas.environment.os: unknown, + aas.environment.runtime: .NET, + aas.site.kind: functionapp, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + aspnet_core.endpoint: BadRequest, + aspnet_core.route: api/badrequest, + component: aspnet_core, + env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/badrequest, + http.status_code: 400, + http.url: http://localhost:00000/api/badrequest, + language: dotnet, + runtime-id: Guid_1, + span.kind: server + }, + Metrics: { + process_id: 0, + _dd.top_level: 1.0, + _dd.tracer_kr: 1.0, + _sampling_priority_v1: 1.0 + } + }, + { + TraceId: Id_1, + SpanId: Id_4, Name: azure_functions.invoke, - Resource: Timer TriggerAllTimer, + Resource: Http BadRequest, Service: AzureFunctionsAllTriggers, Type: serverless, + ParentId: Id_2, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, + aas.function.name: BadRequest, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_5, + Name: Manual inside BadRequest, + Resource: Manual inside BadRequest, + Service: AzureFunctionsAllTriggers, + ParentId: Id_4, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_6, + Name: aspnet_core.request, + Resource: GET /api/error, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_7, + Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, - aas.function.name: TriggerAllTimer, - aas.function.trigger: Timer, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: ServerError, + aspnet_core.route: api/error, + component: aspnet_core, env: integration_tests, + error.msg: The HTTP response has status code 500., + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/error, + http.status_code: 500, + http.url: http://localhost:00000/api/error, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -33,60 +111,143 @@ }, { TraceId: Id_1, - SpanId: Id_3, - Name: http.request, - Resource: GET localhost:00000/api/trigger, - Service: AzureFunctionsAllTriggers-http-client, - Type: http, - ParentId: Id_2, + SpanId: Id_8, + Name: azure_functions.invoke, + Resource: Http ServerError, + Service: AzureFunctionsAllTriggers, + Type: serverless, + ParentId: Id_6, Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, + aas.function.name: ServerError, + aas.function.trigger: Http, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: HttpMessageHandler, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_9, + Name: Manual inside ServerError, + Resource: Manual inside ServerError, + Service: AzureFunctionsAllTriggers, + ParentId: Id_8, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_10, + Name: aspnet_core.request, + Resource: GET /api/exception, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_11, + Tags: { + aas.environment.extension_version: unknown, + aas.environment.instance_id: unknown, + aas.environment.instance_name: IntegrationTestHost, + aas.environment.os: unknown, + aas.environment.runtime: .NET, + aas.site.kind: functionapp, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + aspnet_core.endpoint: Exception, + aspnet_core.route: api/exception, + component: aspnet_core, env: integration_tests, - http-client-handler-type: System.Net.Http.HttpClientHandler, http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/exception, http.status_code: 200, - http.url: http://localhost:00000/api/trigger, + http.url: http://localhost:00000/api/exception, language: dotnet, - out.host: localhost, runtime-id: Guid_1, - span.kind: client, - _dd.base_service: AzureFunctionsAllTriggers, - _dd.svc_src: http-client + span.kind: server }, Metrics: { - _dd.top_level: 1.0 + process_id: 0, + _dd.top_level: 1.0, + _dd.tracer_kr: 1.0, + _sampling_priority_v1: 1.0 } }, { TraceId: Id_1, - SpanId: Id_4, + SpanId: Id_12, Name: azure_functions.invoke, - Resource: GET /api/trigger, + Resource: Http Exception, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_3, + ParentId: Id_10, + Error: 1, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, + aas.function.name: Exception, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + error.msg: Task failed successfully., + error.stack: +System.InvalidOperationException: Task failed successfully. +at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), + error.type: System.InvalidOperationException, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_13, + Name: Manual inside Exception, + Resource: Manual inside Exception, + Service: AzureFunctionsAllTriggers, + ParentId: Id_12, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_14, + Name: aspnet_core.request, + Resource: GET /api/simple, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_15, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.name: TriggerCaller, - aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: SimpleHttpTrigger, + aspnet_core.route: api/simple, + component: aspnet_core, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, + http.route: api/simple, http.status_code: 200, - http.url: http://localhost:00000/api/trigger, + http.url: http://localhost:00000/api/simple, language: dotnet, - runtime-id: Guid_2, + runtime-id: Guid_1, span.kind: server }, Metrics: { @@ -98,26 +259,64 @@ }, { TraceId: Id_1, - SpanId: Id_5, + SpanId: Id_16, Name: azure_functions.invoke, - Resource: Http TriggerCaller, + Resource: Http SimpleHttpTrigger, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_4, + ParentId: Id_14, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, + aas.function.name: SimpleHttpTrigger, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_17, + Name: Manual inside Simple, + Resource: Manual inside Simple, + Service: AzureFunctionsAllTriggers, + ParentId: Id_16, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_18, + Name: aspnet_core.request, + Resource: GET /api/trigger, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_19, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, - aas.function.name: TriggerCaller, - aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: TriggerCaller, + aspnet_core.route: api/trigger, + component: aspnet_core, env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/trigger, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -131,11 +330,31 @@ }, { TraceId: Id_1, - SpanId: Id_6, + SpanId: Id_20, + Name: azure_functions.invoke, + Resource: Http TriggerCaller, + Service: AzureFunctionsAllTriggers, + Type: serverless, + ParentId: Id_18, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, + aas.function.name: TriggerCaller, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_21, Name: Manual inside Trigger, Resource: Manual inside Trigger, Service: AzureFunctionsAllTriggers, - ParentId: Id_5, + ParentId: Id_20, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -145,12 +364,12 @@ }, { TraceId: Id_1, - SpanId: Id_7, + SpanId: Id_22, Name: http.request, Resource: GET localhost:00000/api/simple, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -173,12 +392,12 @@ }, { TraceId: Id_1, - SpanId: Id_8, + SpanId: Id_23, Name: http.request, Resource: GET localhost:00000/api/exception, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -201,12 +420,12 @@ }, { TraceId: Id_1, - SpanId: Id_9, + SpanId: Id_24, Name: http.request, Resource: GET localhost:00000/api/error, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -229,12 +448,12 @@ }, { TraceId: Id_1, - SpanId: Id_10, + SpanId: Id_25, Name: http.request, Resource: GET localhost:00000/api/badrequest, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Error: 1, Tags: { aas.site.name: AzureFunctionsAllTriggers, @@ -259,12 +478,12 @@ }, { TraceId: Id_1, - SpanId: Id_11, + SpanId: Id_26, Name: azure_functions.invoke, Resource: GET /api/simple, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_7, + ParentId: Id_22, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -296,12 +515,12 @@ }, { TraceId: Id_1, - SpanId: Id_12, + SpanId: Id_27, Name: azure_functions.invoke, Resource: GET /api/exception, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_8, + ParentId: Id_23, Error: 1, Tags: { aas.environment.extension_version: unknown, @@ -342,12 +561,12 @@ at Samples.AzureFunctions.V4Isolated.AspNetCore.DirectFunctionExecutor.ExecuteAs }, { TraceId: Id_1, - SpanId: Id_13, + SpanId: Id_28, Name: azure_functions.invoke, Resource: GET /api/error, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_9, + ParentId: Id_24, Error: 1, Tags: { aas.environment.extension_version: unknown, @@ -381,12 +600,12 @@ at Samples.AzureFunctions.V4Isolated.AspNetCore.DirectFunctionExecutor.ExecuteAs }, { TraceId: Id_1, - SpanId: Id_14, + SpanId: Id_29, Name: azure_functions.invoke, Resource: GET /api/badrequest, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_10, + ParentId: Id_25, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -418,65 +637,25 @@ at Samples.AzureFunctions.V4Isolated.AspNetCore.DirectFunctionExecutor.ExecuteAs }, { TraceId: Id_1, - SpanId: Id_15, - Name: azure_functions.invoke, - Resource: Http SimpleHttpTrigger, - Service: AzureFunctionsAllTriggers, - Type: serverless, - ParentId: Id_11, - Tags: { - aas.environment.extension_version: unknown, - aas.environment.instance_id: unknown, - aas.environment.instance_name: IntegrationTestHost, - aas.environment.os: unknown, - aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, - aas.function.name: SimpleHttpTrigger, - aas.function.trigger: Http, - aas.site.kind: functionapp, - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - component: AzureFunctions, - env: integration_tests, - language: dotnet, - runtime-id: Guid_1, - span.kind: server - }, - Metrics: { - process_id: 0, - _dd.top_level: 1.0, - _dd.tracer_kr: 1.0, - _sampling_priority_v1: 1.0 - } - }, - { - TraceId: Id_1, - SpanId: Id_16, + SpanId: Id_30, Name: azure_functions.invoke, - Resource: Http Exception, + Resource: Timer TriggerAllTimer, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_12, - Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, - aas.function.name: Exception, - aas.function.trigger: Http, + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, + aas.function.name: TriggerAllTimer, + aas.function.trigger: Timer, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, component: AzureFunctions, env: integration_tests, - error.msg: Task failed successfully., - error.stack: -System.InvalidOperationException: Task failed successfully. -at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), - error.type: System.InvalidOperationException, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -490,61 +669,59 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) }, { TraceId: Id_1, - SpanId: Id_17, - Name: azure_functions.invoke, - Resource: Http ServerError, - Service: AzureFunctionsAllTriggers, - Type: serverless, - ParentId: Id_13, + SpanId: Id_31, + Name: http.request, + Resource: GET localhost:00000/api/trigger, + Service: AzureFunctionsAllTriggers-http-client, + Type: http, + ParentId: Id_30, Tags: { - aas.environment.extension_version: unknown, - aas.environment.instance_id: unknown, - aas.environment.instance_name: IntegrationTestHost, - aas.environment.os: unknown, - aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, - aas.function.name: ServerError, - aas.function.trigger: Http, - aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: HttpMessageHandler, env: integration_tests, + http-client-handler-type: System.Net.Http.HttpClientHandler, + http.method: GET, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, + out.host: localhost, runtime-id: Guid_1, - span.kind: server + span.kind: client, + _dd.base_service: AzureFunctionsAllTriggers }, Metrics: { - process_id: 0, - _dd.top_level: 1.0, - _dd.tracer_kr: 1.0, - _sampling_priority_v1: 1.0 + _dd.top_level: 1.0 } }, { TraceId: Id_1, - SpanId: Id_18, + SpanId: Id_32, Name: azure_functions.invoke, - Resource: Http BadRequest, + Resource: GET /api/trigger, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_14, + ParentId: Id_31, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, - aas.function.name: BadRequest, + aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.name: TriggerCaller, aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, component: AzureFunctions, env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, - runtime-id: Guid_1, + runtime-id: Guid_2, span.kind: server }, Metrics: { @@ -553,61 +730,5 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) _dd.tracer_kr: 1.0, _sampling_priority_v1: 1.0 } - }, - { - TraceId: Id_1, - SpanId: Id_19, - Name: Manual inside Simple, - Resource: Manual inside Simple, - Service: AzureFunctionsAllTriggers, - ParentId: Id_15, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_20, - Name: Manual inside Exception, - Resource: Manual inside Exception, - Service: AzureFunctionsAllTriggers, - ParentId: Id_16, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_21, - Name: Manual inside ServerError, - Resource: Manual inside ServerError, - Service: AzureFunctionsAllTriggers, - ParentId: Id_17, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_22, - Name: Manual inside BadRequest, - Resource: Manual inside BadRequest, - Service: AzureFunctionsAllTriggers, - ParentId: Id_18, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } } ] \ No newline at end of file diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt index 5ef6fca4b60e..aabaf13c7a35 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt @@ -2,24 +2,102 @@ { TraceId: Id_1, SpanId: Id_2, + Name: aspnet_core.request, + Resource: GET /api/badrequest, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_3, + Tags: { + aas.environment.extension_version: unknown, + aas.environment.instance_id: unknown, + aas.environment.instance_name: IntegrationTestHost, + aas.environment.os: unknown, + aas.environment.runtime: .NET, + aas.site.kind: functionapp, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + aspnet_core.endpoint: BadRequest, + aspnet_core.route: api/badrequest, + component: aspnet_core, + env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/badrequest, + http.status_code: 400, + http.url: http://localhost:00000/api/badrequest, + language: dotnet, + runtime-id: Guid_1, + span.kind: server + }, + Metrics: { + process_id: 0, + _dd.top_level: 1.0, + _dd.tracer_kr: 1.0, + _sampling_priority_v1: 1.0 + } + }, + { + TraceId: Id_1, + SpanId: Id_4, Name: azure_functions.invoke, - Resource: Timer TriggerAllTimer, + Resource: Http BadRequest, Service: AzureFunctionsAllTriggers, Type: serverless, + ParentId: Id_2, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, + aas.function.name: BadRequest, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_5, + Name: Manual inside BadRequest, + Resource: Manual inside BadRequest, + Service: AzureFunctionsAllTriggers, + ParentId: Id_4, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_6, + Name: aspnet_core.request, + Resource: GET /api/error, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_7, + Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, - aas.function.name: TriggerAllTimer, - aas.function.trigger: Timer, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: ServerError, + aspnet_core.route: api/error, + component: aspnet_core, env: integration_tests, + error.msg: The HTTP response has status code 500., + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/error, + http.status_code: 500, + http.url: http://localhost:00000/api/error, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -33,60 +111,143 @@ }, { TraceId: Id_1, - SpanId: Id_3, - Name: http.request, - Resource: GET localhost:00000/api/trigger, - Service: AzureFunctionsAllTriggers-http-client, - Type: http, - ParentId: Id_2, + SpanId: Id_8, + Name: azure_functions.invoke, + Resource: Http ServerError, + Service: AzureFunctionsAllTriggers, + Type: serverless, + ParentId: Id_6, Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, + aas.function.name: ServerError, + aas.function.trigger: Http, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: HttpMessageHandler, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_9, + Name: Manual inside ServerError, + Resource: Manual inside ServerError, + Service: AzureFunctionsAllTriggers, + ParentId: Id_8, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_10, + Name: aspnet_core.request, + Resource: GET /api/exception, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_11, + Tags: { + aas.environment.extension_version: unknown, + aas.environment.instance_id: unknown, + aas.environment.instance_name: IntegrationTestHost, + aas.environment.os: unknown, + aas.environment.runtime: .NET, + aas.site.kind: functionapp, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + aspnet_core.endpoint: Exception, + aspnet_core.route: api/exception, + component: aspnet_core, env: integration_tests, - http-client-handler-type: System.Net.Http.HttpClientHandler, http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/exception, http.status_code: 200, - http.url: http://localhost:00000/api/trigger, + http.url: http://localhost:00000/api/exception, language: dotnet, - out.host: localhost, runtime-id: Guid_1, - span.kind: client, - _dd.base_service: AzureFunctionsAllTriggers, - _dd.svc_src: http-client + span.kind: server }, Metrics: { - _dd.top_level: 1.0 + process_id: 0, + _dd.top_level: 1.0, + _dd.tracer_kr: 1.0, + _sampling_priority_v1: 1.0 } }, { TraceId: Id_1, - SpanId: Id_4, + SpanId: Id_12, Name: azure_functions.invoke, - Resource: GET /api/trigger, + Resource: Http Exception, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_3, + ParentId: Id_10, + Error: 1, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, + aas.function.name: Exception, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + error.msg: Task failed successfully., + error.stack: +System.InvalidOperationException: Task failed successfully. +at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), + error.type: System.InvalidOperationException, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_13, + Name: Manual inside Exception, + Resource: Manual inside Exception, + Service: AzureFunctionsAllTriggers, + ParentId: Id_12, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_14, + Name: aspnet_core.request, + Resource: GET /api/simple, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_15, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.name: TriggerCaller, - aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: SimpleHttpTrigger, + aspnet_core.route: api/simple, + component: aspnet_core, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, + http.route: api/simple, http.status_code: 200, - http.url: http://localhost:00000/api/trigger, + http.url: http://localhost:00000/api/simple, language: dotnet, - runtime-id: Guid_2, + runtime-id: Guid_1, span.kind: server }, Metrics: { @@ -98,26 +259,64 @@ }, { TraceId: Id_1, - SpanId: Id_5, + SpanId: Id_16, Name: azure_functions.invoke, - Resource: Http TriggerCaller, + Resource: Http SimpleHttpTrigger, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_4, + ParentId: Id_14, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, + aas.function.name: SimpleHttpTrigger, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_17, + Name: Manual inside Simple, + Resource: Manual inside Simple, + Service: AzureFunctionsAllTriggers, + ParentId: Id_16, + Tags: { + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + env: integration_tests, + language: dotnet + } + }, + { + TraceId: Id_1, + SpanId: Id_18, + Name: aspnet_core.request, + Resource: GET /api/trigger, + Service: AzureFunctionsAllTriggers, + Type: web, + ParentId: Id_19, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, - aas.function.name: TriggerCaller, - aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + aspnet_core.endpoint: TriggerCaller, + aspnet_core.route: api/trigger, + component: aspnet_core, env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.route: api/trigger, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -131,11 +330,31 @@ }, { TraceId: Id_1, - SpanId: Id_6, + SpanId: Id_20, + Name: azure_functions.invoke, + Resource: Http TriggerCaller, + Service: AzureFunctionsAllTriggers, + Type: serverless, + ParentId: Id_18, + Tags: { + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, + aas.function.name: TriggerCaller, + aas.function.trigger: Http, + aas.site.name: AzureFunctionsAllTriggers, + aas.site.type: function, + component: AzureFunctions, + env: integration_tests, + language: dotnet, + span.kind: server + } + }, + { + TraceId: Id_1, + SpanId: Id_21, Name: Manual inside Trigger, Resource: Manual inside Trigger, Service: AzureFunctionsAllTriggers, - ParentId: Id_5, + ParentId: Id_20, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -145,12 +364,12 @@ }, { TraceId: Id_1, - SpanId: Id_7, + SpanId: Id_22, Name: http.request, Resource: GET localhost:00000/api/simple, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -173,12 +392,12 @@ }, { TraceId: Id_1, - SpanId: Id_8, + SpanId: Id_23, Name: http.request, Resource: GET localhost:00000/api/exception, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -201,12 +420,12 @@ }, { TraceId: Id_1, - SpanId: Id_9, + SpanId: Id_24, Name: http.request, Resource: GET localhost:00000/api/error, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Tags: { aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -229,12 +448,12 @@ }, { TraceId: Id_1, - SpanId: Id_10, + SpanId: Id_25, Name: http.request, Resource: GET localhost:00000/api/badrequest, Service: AzureFunctionsAllTriggers-http-client, Type: http, - ParentId: Id_6, + ParentId: Id_21, Error: 1, Tags: { aas.site.name: AzureFunctionsAllTriggers, @@ -259,12 +478,12 @@ }, { TraceId: Id_1, - SpanId: Id_11, + SpanId: Id_26, Name: azure_functions.invoke, Resource: GET /api/simple, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_7, + ParentId: Id_22, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -296,12 +515,12 @@ }, { TraceId: Id_1, - SpanId: Id_12, + SpanId: Id_27, Name: azure_functions.invoke, Resource: GET /api/exception, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_8, + ParentId: Id_23, Error: 1, Tags: { aas.environment.extension_version: unknown, @@ -341,12 +560,12 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) }, { TraceId: Id_1, - SpanId: Id_13, + SpanId: Id_28, Name: azure_functions.invoke, Resource: GET /api/error, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_9, + ParentId: Id_24, Error: 1, Tags: { aas.environment.extension_version: unknown, @@ -380,12 +599,12 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) }, { TraceId: Id_1, - SpanId: Id_14, + SpanId: Id_29, Name: azure_functions.invoke, Resource: GET /api/badrequest, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_10, + ParentId: Id_25, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -417,65 +636,25 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) }, { TraceId: Id_1, - SpanId: Id_15, - Name: azure_functions.invoke, - Resource: Http SimpleHttpTrigger, - Service: AzureFunctionsAllTriggers, - Type: serverless, - ParentId: Id_11, - Tags: { - aas.environment.extension_version: unknown, - aas.environment.instance_id: unknown, - aas.environment.instance_name: IntegrationTestHost, - aas.environment.os: unknown, - aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, - aas.function.name: SimpleHttpTrigger, - aas.function.trigger: Http, - aas.site.kind: functionapp, - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - component: AzureFunctions, - env: integration_tests, - language: dotnet, - runtime-id: Guid_1, - span.kind: server - }, - Metrics: { - process_id: 0, - _dd.top_level: 1.0, - _dd.tracer_kr: 1.0, - _sampling_priority_v1: 1.0 - } - }, - { - TraceId: Id_1, - SpanId: Id_16, + SpanId: Id_30, Name: azure_functions.invoke, - Resource: Http Exception, + Resource: Timer TriggerAllTimer, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_12, - Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, - aas.function.name: Exception, - aas.function.trigger: Http, + aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, + aas.function.name: TriggerAllTimer, + aas.function.trigger: Timer, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, component: AzureFunctions, env: integration_tests, - error.msg: Task failed successfully., - error.stack: -System.InvalidOperationException: Task failed successfully. -at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), - error.type: System.InvalidOperationException, language: dotnet, runtime-id: Guid_1, span.kind: server @@ -489,61 +668,59 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) }, { TraceId: Id_1, - SpanId: Id_17, - Name: azure_functions.invoke, - Resource: Http ServerError, - Service: AzureFunctionsAllTriggers, - Type: serverless, - ParentId: Id_13, + SpanId: Id_31, + Name: http.request, + Resource: GET localhost:00000/api/trigger, + Service: AzureFunctionsAllTriggers-http-client, + Type: http, + ParentId: Id_30, Tags: { - aas.environment.extension_version: unknown, - aas.environment.instance_id: unknown, - aas.environment.instance_name: IntegrationTestHost, - aas.environment.os: unknown, - aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, - aas.function.name: ServerError, - aas.function.trigger: Http, - aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: AzureFunctions, + component: HttpMessageHandler, env: integration_tests, + http-client-handler-type: System.Net.Http.HttpClientHandler, + http.method: GET, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, + out.host: localhost, runtime-id: Guid_1, - span.kind: server + span.kind: client, + _dd.base_service: AzureFunctionsAllTriggers }, Metrics: { - process_id: 0, - _dd.top_level: 1.0, - _dd.tracer_kr: 1.0, - _sampling_priority_v1: 1.0 + _dd.top_level: 1.0 } }, { TraceId: Id_1, - SpanId: Id_18, + SpanId: Id_32, Name: azure_functions.invoke, - Resource: Http BadRequest, + Resource: GET /api/trigger, Service: AzureFunctionsAllTriggers, Type: serverless, - ParentId: Id_14, + ParentId: Id_31, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, aas.environment.instance_name: IntegrationTestHost, aas.environment.os: unknown, aas.environment.runtime: .NET, - aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, - aas.function.name: BadRequest, + aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, + aas.function.name: TriggerCaller, aas.function.trigger: Http, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, component: AzureFunctions, env: integration_tests, + http.method: GET, + http.request.headers.host: localhost:00000, + http.status_code: 200, + http.url: http://localhost:00000/api/trigger, language: dotnet, - runtime-id: Guid_1, + runtime-id: Guid_2, span.kind: server }, Metrics: { @@ -552,61 +729,5 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) _dd.tracer_kr: 1.0, _sampling_priority_v1: 1.0 } - }, - { - TraceId: Id_1, - SpanId: Id_19, - Name: Manual inside Simple, - Resource: Manual inside Simple, - Service: AzureFunctionsAllTriggers, - ParentId: Id_15, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_20, - Name: Manual inside Exception, - Resource: Manual inside Exception, - Service: AzureFunctionsAllTriggers, - ParentId: Id_16, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_21, - Name: Manual inside ServerError, - Resource: Manual inside ServerError, - Service: AzureFunctionsAllTriggers, - ParentId: Id_17, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } - }, - { - TraceId: Id_1, - SpanId: Id_22, - Name: Manual inside BadRequest, - Resource: Manual inside BadRequest, - Service: AzureFunctionsAllTriggers, - ParentId: Id_18, - Tags: { - aas.site.name: AzureFunctionsAllTriggers, - aas.site.type: function, - env: integration_tests, - language: dotnet - } } ] \ No newline at end of file From c700ea4c14cd8e35a4b492fe51fdcf489c0e62a4 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 12 Dec 2025 17:49:25 -0500 Subject: [PATCH 05/42] only apply this fix to azure functions using extension v4 --- tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index 3f32a82f6567..863ded7ca8b8 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -486,7 +486,10 @@ private static void InitializeTracer(ref RefStopwatch sw) private static void StartDiagnosticManager() { - var observers = new List(); + var observers = new List + { + new QuartzDiagnosticObserver() + }; #if !NETFRAMEWORK if (!SkipAspNetCoreDiagnosticObserver()) From c632a1ad42dd4e09c2f610da6453a9e1f6e48bbb Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 16 Dec 2025 15:28:13 -0500 Subject: [PATCH 06/42] refactor AspNetCoreDiagObserver initialization --- .../ClrProfiler/Instrumentation.cs | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index 863ded7ca8b8..c1681b67a466 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -486,10 +486,7 @@ private static void InitializeTracer(ref RefStopwatch sw) private static void StartDiagnosticManager() { - var observers = new List - { - new QuartzDiagnosticObserver() - }; + var observers = new List(); #if !NETFRAMEWORK if (!SkipAspNetCoreDiagnosticObserver()) @@ -528,10 +525,50 @@ private static AspNetCoreDiagnosticObserver GetAspNetCoreDiagnosticObserver() [Pure] private static bool SkipAspNetCoreDiagnosticObserver() { - // this is extremely simple now, but will get more complex soon... - return AzureInfo.Instance.IsAzureFunction; + // Enable AspNetCoreDiagnosticObserver in: + // - outside Azure Functions + // - Isolated functions worker processes with extension v4 + // (to create aspnet_core.request spans that azure_functions.invoke can parent to) + + // Skip AspNetCoreDiagnosticObserver in Azure Functions: + // - In-process functions (due to AssemblyLoadContext issues) + // - Isolated functions host process (to avoid duplicate spans) + // - Isolated functions worker process with extension v1 (FUNCTIONS_EXTENSION_VERSION="~1") + + if (!AzureInfo.Instance.IsAzureFunction) + { + // we only need to skip in some Azure Functions + return false; + } + + if (AzureInfo.Instance.IsIsolatedFunctionHostProcess) + { + // Skip AspNetCoreDiagnosticObserver in Azure Functions _host_ processes + Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in an isolated Azure Function host process."); + return true; + } + + if (!AzureInfo.Instance.IsIsolatedFunction) + { + // Skip AspNetCoreDiagnosticObserver in in-process Azure Functions + Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in an in-process Azure Function."); + return true; + } + + // FUNCTIONS_EXTENSION_VERSION + var azureFunctionsExtensionVersion = AzureInfo.Instance.AzureFunctionsExtensionVersion; + + if (azureFunctionsExtensionVersion != "~4") + { + // Skip AspNetCoreDiagnosticObserver in v1 functions (v2 and v3 are not supported at all) + Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in Azure Function with extension version {AzureFunctionsExtensionVersion}.", azureFunctionsExtensionVersion); + return true; + } + + // do not skip when running in an isolated Azure Functions worker process with extension v4 + return false; } -#endif +#endif // #if !NETFRAMEWORK private static void InitializeDebugger(TracerSettings tracerSettings) { From 6a128251c58bd079958deac89924e1b271f47a40 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 13 Jan 2026 15:39:07 -0500 Subject: [PATCH 07/42] check if logging is enabled --- .../Azure/Functions/AzureFunctionsCommon.cs | 12 +++++++++++- .../AspNetCoreHttpRequestHandler.cs | 19 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 6f02fa65a019..f7e205c05244 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -22,6 +22,7 @@ using Datadog.Trace.Util; using Datadog.Trace.Util.Json; using Datadog.Trace.Vendors.Newtonsoft.Json; +using Datadog.Trace.Vendors.Serilog.Events; #nullable enable @@ -363,7 +364,16 @@ httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && scopeObj is Scope aspNetCoreScope) { parentScope = aspNetCoreScope; - Log.Debug("Azure Functions span creation: Retrieved AspNetCore scope - span_id: {SpanId}, trace_id: {TraceId}", aspNetCoreScope.Span.SpanId, aspNetCoreScope.Span.TraceId); + + if (Log.IsEnabled(LogEventLevel.Debug)) + { + var spanContext = parentScope.Span.Context; + + Log.Debug( + "Azure Functions span creation: Retrieved AspNetCore scope {TraceId}-{SpanId}", + spanContext.RawTraceId, + spanContext.RawSpanId); + } } else { diff --git a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs index 17c227845357..c221ce2a43c6 100644 --- a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs +++ b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs @@ -28,6 +28,7 @@ using Datadog.Trace.Tagging; using Datadog.Trace.Util; using Datadog.Trace.Util.Http; +using Datadog.Trace.Vendors.Serilog.Events; using Microsoft.AspNetCore.Http; namespace Datadog.Trace.PlatformHelpers @@ -174,10 +175,20 @@ private Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, Ias httpContext.Items[HttpContextTrackingKey] = new RequestTrackingFeature(originalPath, scope, proxyContext?.Scope); #endif - // Store scope in HttpContext.Items for Azure Functions middleware to retrieve - // Use __ prefix to avoid conflicts with user code (same pattern as TracingHttpModule) - httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope; - _log.Debug("AspNetCore: Stored scope in HttpContext.Items - span_id: {SpanId}, trace_id: {TraceId}, path: {Path}", scope.Span.SpanId, scope.Span.TraceId, request.Path); + if (EnvironmentHelpers.IsAzureFunctions()) + { + // Store scope in HttpContext.Items for Azure Functions middleware to retrieve + httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope; + + if (_log.IsEnabled(LogEventLevel.Debug) && scope.Span.Context is { } spanContext) + { + _log.Debug( + "AspNetCore: Stored scope in HttpContext.Items, {TraceId}-{SpanId}, path: {Path}", + spanContext.RawTraceId, + spanContext.RawSpanId, + request.Path); + } + } if (tracer.Settings.IpHeaderEnabled || security.AppsecEnabled) { From 95fb93c36e27fae5680ea118ae6319355f8aa8c9 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 13 Jan 2026 15:40:24 -0500 Subject: [PATCH 08/42] use new HttpContextActiveScopeKey constant --- .../Azure/Functions/AzureFunctionsCommon.cs | 3 ++- .../PlatformHelpers/AspNetCoreHttpRequestHandler.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index f7e205c05244..41650e9ad4c7 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -17,6 +17,7 @@ using Datadog.Trace.DuckTyping; using Datadog.Trace.Headers; using Datadog.Trace.Logging; +using Datadog.Trace.PlatformHelpers; using Datadog.Trace.Propagators; using Datadog.Trace.Tagging; using Datadog.Trace.Util; @@ -360,7 +361,7 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even if (functionContext.Items != null && functionContext.Items.TryGetValue("HttpRequestContext", out var httpContextObj) && httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && - httpContext.Items.TryGetValue("__Datadog.Trace.AspNetCore.ActiveScope", out var scopeObj) && + httpContext.Items.TryGetValue(AspNetCoreHttpRequestHandler.HttpContextActiveScopeKey, out var scopeObj) && scopeObj is Scope aspNetCoreScope) { parentScope = aspNetCoreScope; diff --git a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs index c221ce2a43c6..8721756ce468 100644 --- a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs +++ b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs @@ -36,6 +36,7 @@ namespace Datadog.Trace.PlatformHelpers internal sealed class AspNetCoreHttpRequestHandler { internal const string HttpContextTrackingKey = "__Datadog.AspNetCoreHttpRequestHandler.Tracking"; + internal const string HttpContextActiveScopeKey = "__Datadog.AspNetCoreHttpRequestHandler.ActiveScope"; private readonly IDatadogLogger _log; private readonly IntegrationId _integrationId; @@ -178,7 +179,7 @@ private Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, Ias if (EnvironmentHelpers.IsAzureFunctions()) { // Store scope in HttpContext.Items for Azure Functions middleware to retrieve - httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope; + httpContext.Items[HttpContextActiveScopeKey] = scope; if (_log.IsEnabled(LogEventLevel.Debug) && scope.Span.Context is { } spanContext) { From a5a5ac658e851c0549b2147da32f626fc12eb004 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 13 Nov 2025 14:47:38 -0500 Subject: [PATCH 09/42] delete temporary investigation files --- .../trace_payload_1762897459231.json | 1 - .../trace_payload_1762897460146.json | 1 - .../trace_payload_1762824450957.json | 1 - .../trace_payload_1762824451432.json | 1 - ...-981_ab644f9bfc6f421a8b58d26611dbfdde.json | 1 - ...-203_e4e0746e72de473dac6050a278cb36fd.json | 1 - ...MSVLS-58-Azure-Functions-span-parenting.md | 395 ------------------ 7 files changed, 401 deletions(-) delete mode 100644 docs/development/investigations/1-original/trace_payload_1762897459231.json delete mode 100644 docs/development/investigations/1-original/trace_payload_1762897460146.json delete mode 100644 docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json delete mode 100644 docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json delete mode 100644 docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json delete mode 100644 docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json delete mode 100644 docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md diff --git a/docs/development/investigations/1-original/trace_payload_1762897459231.json b/docs/development/investigations/1-original/trace_payload_1762897459231.json deleted file mode 100644 index 9657a3265ebe..000000000000 --- a/docs/development/investigations/1-original/trace_payload_1762897459231.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":1910270346618876437,"span_id":4052593309550270339,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762897459030835400,"duration":115514400,"parent_id":13925130946451077401,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6913ae3200000000","runtime-id":"3e375b7a-5170-460c-bd77-278b812ca542","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":1910270346618876437,"span_id":13925130946451077401,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762897459015207300,"duration":135993900,"parent_id":18286165385944622934,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":1910270346618876437,"span_id":18286165385944622934,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762897458993111500,"duration":159499100,"parent_id":1978509666546725896,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"3e375b7a-5170-460c-bd77-278b812ca542","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/1-original/trace_payload_1762897460146.json b/docs/development/investigations/1-original/trace_payload_1762897460146.json deleted file mode 100644 index 56f0aac1c7f6..000000000000 --- a/docs/development/investigations/1-original/trace_payload_1762897460146.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":1910270346618876437,"span_id":5775791465145012166,"name":"http.request","resource":"GET localhost:43239/api/HttpTest","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762897458789654200,"duration":413406600,"parent_id":1978509666546725896,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"http://localhost:43239/api/HttpTest","http-client-handler-type":"System.Net.Http.SocketsHttpHandler","http.status_code":"200","out.host":"localhost","_dd.p.dm":"-1","_dd.p.tid":"6913ae3200000000","runtime-id":"1ecec7fe-9bf8-4cd7-8a14-f47f80add4a8","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"b358dabba967e43e3c9b5e85bd392313864792a2","_dd.git.repository_url":"https://github.com/Azure/azure-functions-host","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"_dd.top_level":1}},{"trace_id":1910270346618876437,"span_id":1978509666546725896,"name":"azure_functions.invoke","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762897458672838500,"duration":635318800,"meta":{"component":"AzureFunctions","span.kind":"server","http.useragent":"curl/8.16.0","http.method":"GET","http.request.headers.host":"lucasp-premium-linux-isolated-aspnet.azurewebsites.net","http.url":"https://lucasp-premium-linux-isolated-aspnet.azurewebsites.net/api/HttpTest","http.status_code":"200","aas.function.name":"HttpTest","aas.function.binding":"Microsoft.Azure.WebJobs.Host.Executors.BindingSource","aas.function.trigger":"Http","runtime-id":"1ecec7fe-9bf8-4cd7-8a14-f47f80add4a8","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"process_id":27,"_dd.agent_psr":1,"_dd.tracer_kr":1,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json deleted file mode 100644 index 7e035b1a6be1..000000000000 --- a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":11227614026327825028,"span_id":3342051758595645288,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762824450609760600,"duration":170438800,"parent_id":9984891158266111958,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":11227614026327825028,"span_id":9984891158266111958,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762824450594507000,"duration":190324400,"parent_id":727838728754785615,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":11227614026327825028,"span_id":727838728754785615,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762824450568570100,"duration":217483300,"parent_id":12539548146409755932,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}],[{"trace_id":11227614026327825028,"span_id":5312880851904873807,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762824450422449400,"duration":420572900,"parent_id":17651134856053342758,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"localhost:43229","http.url":"http://localhost:43229/api/HttpTest","http.status_code":"200","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"39f899cb-ef3a-4121-bf12-d902cf00ee2c","env":"lucas.pimentel","language":"dotnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json b/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json deleted file mode 100644 index b32d179a2e56..000000000000 --- a/docs/development/investigations/2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":11227614026327825028,"span_id":17651134856053342758,"name":"http.request","resource":"GET localhost:43229/api/HttpTest","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762824450339164400,"duration":497064700,"parent_id":12539548146409755932,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"http://localhost:43229/api/HttpTest","http-client-handler-type":"System.Net.Http.SocketsHttpHandler","http.status_code":"200","out.host":"localhost","_dd.p.dm":"-1","_dd.p.tid":"6912910200000000","runtime-id":"5210fd98-a24d-4204-9f43-118a5fa2c857","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"b358dabba967e43e3c9b5e85bd392313864792a2","_dd.git.repository_url":"https://github.com/Azure/azure-functions-host","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"_dd.top_level":1}},{"trace_id":11227614026327825028,"span_id":12539548146409755932,"name":"azure_functions.invoke","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762824450215590500,"duration":784896500,"meta":{"component":"AzureFunctions","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"lucasp-premium-linux-isolated-aspnet.azurewebsites.net","http.url":"https://lucasp-premium-linux-isolated-aspnet.azurewebsites.net/api/HttpTest","http.status_code":"200","aas.function.name":"HttpTest","aas.function.binding":"Microsoft.Azure.WebJobs.Host.Executors.BindingSource","aas.function.trigger":"Http","runtime-id":"5210fd98-a24d-4204-9f43-118a5fa2c857","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucasp-premium-linux-isolated-aspnet_group","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucasp-premium-linux-isolated-aspnet_group/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"host"},"metrics":{"process_id":27,"_dd.agent_psr":1,"_dd.tracer_kr":1,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json deleted file mode 100644 index c3be8a9f6525..000000000000 --- a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_20-47-48-981_ab644f9bfc6f421a8b58d26611dbfdde.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":17961676107601563275,"span_id":4768510696171411833,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762980468406397700,"duration":473585100,"parent_id":7380191810641356982,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"6914f27400000000","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":17961676107601563275,"span_id":7380191810641356982,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762980468391630100,"duration":492717200,"parent_id":15573027893036604679,"meta":{"process.id":"59","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":17961676107601563275,"span_id":15573027893036604679,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762980468368988400,"duration":516924400,"parent_id":4682913969503290245,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":59,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}],[{"trace_id":17961676107601563275,"span_id":4316486117327122103,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762980468236513800,"duration":704521600,"parent_id":243437182453255327,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"xh/0.25.0","http.method":"GET","http.request.headers.host":"localhost:42959","http.url":"http://localhost:42959/api/HttpTest","http.status_code":"200","_dd.p.dm":"-1","_dd.p.tid":"6914f27400000000","runtime-id":"014e39b3-2f77-4776-8e40-1e27353d2443","env":"lucas.pimentel","language":"dotnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":59,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json b/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json deleted file mode 100644 index e8f580d623ed..000000000000 --- a/docs/development/investigations/3-after-using-httpcontext-items/trace_payload_2025-11-12_22-38-15-203_e4e0746e72de473dac6050a278cb36fd.json +++ /dev/null @@ -1 +0,0 @@ -[[{"trace_id":14656220060439490006,"span_id":3397667712105060208,"name":"http.request","resource":"GET jsonplaceholder.typicode.com/users/?","service":"lucasp-premium-linux-isolated-aspnet-http-client","type":"http","start":1762987094553261900,"duration":148844700,"parent_id":11352266471047046875,"meta":{"span.kind":"client","component":"HttpMessageHandler","http.method":"GET","http.url":"https://jsonplaceholder.typicode.com/users/1","http-client-handler-type":"System.Net.Http.HttpClientHandler","http.status_code":"200","out.host":"jsonplaceholder.typicode.com","_dd.p.dm":"-1","_dd.p.tid":"69150c5500000000","runtime-id":"99f59032-46b6-4cf8-8525-0a6d3e7d33f1","env":"lucas.pimentel","language":"dotnet","_dd.base_service":"lucasp-premium-linux-isolated-aspnet","_dd.git.commit.sha":"6563733fd1b07c288045a0c8967b964d463ff767","_dd.git.repository_url":"https://github.com/DataDog/serverless-dev-apps","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"_dd.top_level":1}},{"trace_id":14656220060439490006,"span_id":11352266471047046875,"name":"test_span","resource":"test_span","service":"lucasp-premium-linux-isolated-aspnet","type":null,"start":1762987094530674000,"duration":179029900,"parent_id":1243553223469235235,"meta":{"process.id":"58","process.name":"dotnet","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":14656220060439490006,"span_id":1243553223469235235,"name":"azure_functions.invoke","resource":"Http HttpTest","service":"lucasp-premium-linux-isolated-aspnet","type":"serverless","start":1762987094493292400,"duration":217723200,"parent_id":10174520177415259312,"meta":{"span.kind":"server","component":"AzureFunctions","aas.function.name":"HttpTest","aas.function.method":"AzureFunctions.Isolated.AspNetCore8.Functions.HttpTest","aas.function.trigger":"Http","env":"lucas.pimentel","language":"dotnet","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{}},{"trace_id":14656220060439490006,"span_id":10174520177415259312,"name":"aspnet_core.request","resource":"GET /api/httptest","service":"lucasp-premium-linux-isolated-aspnet","type":"web","start":1762987094280273800,"duration":474975900,"parent_id":17403981112360440532,"meta":{"aspnet_core.endpoint":"HttpTest","component":"aspnet_core","aspnet_core.route":"api/httptest","http.route":"api/httptest","span.kind":"server","http.useragent":"curl/8.16.0","http.method":"GET","http.request.headers.host":"localhost:40813","http.url":"http://localhost:40813/api/HttpTest","http.status_code":"200","runtime-id":"99f59032-46b6-4cf8-8525-0a6d3e7d33f1","env":"lucas.pimentel","language":"dotnet","aas.site.kind":"functionapp","aas.resource.group":"lucas.pimentel","aas.subscription.id":"1dd25961-a5c7-45bf-a5ba-c1475d365cc7","aas.resource.id":"/subscriptions/1dd25961-a5c7-45bf-a5ba-c1475d365cc7/resourcegroups/lucas.pimentel/providers/microsoft.web/sites/lucasp-premium-linux-isolated-aspnet","aas.environment.instance_id":"5093affea35e6b968eed4a112be75fc36be9b8ad0dd0017a95ba7abcac3590e7","aas.environment.instance_name":"pl0sdlwk001D7E","aas.environment.os":"unknown","aas.environment.runtime":".NET","aas.environment.extension_version":"unknown","aas.site.name":"lucasp-premium-linux-isolated-aspnet","aas.site.type":"function","aas.function.extension_version":"~4","aas.function.worker_runtime":"dotnet-isolated","aas.function.process":"worker"},"metrics":{"process_id":58,"_dd.tracer_kr":0,"_sampling_priority_v1":1,"_dd.top_level":1}}]] \ No newline at end of file diff --git a/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md b/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md deleted file mode 100644 index 5cfa90edd61b..000000000000 --- a/docs/development/investigations/APMSVLS-58-Azure-Functions-span-parenting.md +++ /dev/null @@ -1,395 +0,0 @@ -# APMSVLS-58: Azure Functions Span Parenting Issue - -**Status**: ✅ **RESOLVED** (Phase 3 Complete) -**Branch**: `lpimentel/APMSVLS-58-azfunc-host-parenting` -**Date Started**: October 2025 -**Date Resolved**: November 12, 2025 - -## Related Documentation - -- [AzureFunctions.md](../AzureFunctions.md) - Azure Functions integration guide -- [AzureFunctions-Architecture.md](../AzureFunctions-Architecture.md) - Architecture deep dive -- [QueryingDatadogAPIs.md](../QueryingDatadogAPIs.md) - Query Datadog APIs for debugging - -## Problem Description - -In isolated Azure Functions with ASP.NET Core Integration, the worker's `azure_functions.invoke` span is incorrectly parented, resulting in broken distributed traces. - -### Current (Incorrect) Behavior - -**After enabling AspNetCoreDiagnosticObserver in worker** (Phase 2): - -``` -ROOT: azure_functions.invoke [HOST] - └─ http.request [HOST → WORKER] - └─ aspnet_core.request [WORKER] ✓ Correct (now exists!) - -Worker azure_functions.invoke in same trace but WRONG parent: -└─ azure_functions.invoke [WORKER] - └─ parent: host's root span ❌ Should be child of aspnet_core.request - ├─ test_span [WORKER] - └─ http.request [WORKER] -``` - -### Desired Behavior - -``` -ROOT: aspnet_core.request [WORKER] - └─ azure_functions.invoke [WORKER] ✓ Correct - ├─ test_span [WORKER] - └─ http.request [WORKER] - -Note: Host-side azure_functions.invoke and http.request spans should be removed -``` - -**Key observations**: -- All spans tagged with `aas.function.process:host` or `aas.function.process:worker` -- Worker's `azure_functions.invoke` is parented to **host's root span** instead of `aspnet_core.request` -- Need to: (1) Fix worker span parenting (high priority), (2) Remove host-side spans - -## Root Cause (RESOLVED) - -**Primary Issue**: Looking for wrong key in `FunctionContext.Items` -- ❌ Was using: `"__AspNetCoreHttpContext__"` (doesn't exist) -- ✅ Should use: `"HttpRequestContext"` (actual key set by `FunctionsHttpProxyingMiddleware`) - -**Secondary Issue**: Stale header extraction -- In ASP.NET Core mode, headers in gRPC message contain host's root span context -- These stale headers were used as fallback when HttpContext.Items lookup failed -- Worker's `azure_functions.invoke` span incorrectly parented to host's root span - -**Underlying Cause**: AsyncLocal context doesn't flow through Azure Functions middleware, so HttpContext.Items bridge is required. - -## Investigation Findings - -### Confirmed Working ✅ - -1. HTTP proxying: Host returns empty gRPC message when `isHttpProxying` is true -2. HTTP client instrumentation creates `http.request` span and injects trace context headers -3. Worker's `aspnet_core.request` span correctly parented to host's `http.request` span - -### Current Issues ❌ - -1. **AsyncLocal doesn't flow**: `tracer.InternalActiveScope` is null when creating `azure_functions.invoke` span -2. **Activity.Current is broken in Azure Functions**: Cannot rely on `System.Diagnostics.Activity.Current` -3. **Incorrect parent extraction**: Worker's span gets host's root span context instead of `aspnet_core.request` context - -## Trace Analysis - -### Phase 1: Original Behavior (Before AspNetCoreDiagnosticObserver Enabled) - -**Location**: [`1-original/`](1-original/) - -Before enabling AspNetCoreDiagnosticObserver in the worker process, the `aspnet_core.request` span was **not created at all** because the observer was disabled in all Azure Functions processes (both host and worker). - -**Captured traces**: -- [trace_payload_1762897459231.json](1-original/trace_payload_1762897459231.json) - Worker spans -- [trace_payload_1762897460146.json](1-original/trace_payload_1762897460146.json) - Host spans - -**Analysis of trace `1910270346618876437`**: -``` -HOST PROCESS (runtime_id: 1ecec7fe, process: 27): -├─ azure_functions.invoke (span: 1978509666546725896) [ROOT] ❌ -│ resource: "GET /api/httptest" -│ duration: 635ms -│ -└─ http.request (span: 5775791465145012166) ❌ - resource: "GET localhost:43239/api/HttpTest" - duration: 413ms - parent: 1978509666546725896 (host's azure_functions.invoke) - -WORKER PROCESS (runtime_id: 3e375b7a, process: 58): -└─ azure_functions.invoke (span: 18286165385944622934) ⚠️ - resource: "Http HttpTest" - duration: 159ms - parent: 1978509666546725896 (HOST's root span!) - │ - └─ test_span (span: 13925130946451077401) - duration: 136ms - │ - └─ http.request (span: 4052593309550270339) - resource: "GET jsonplaceholder.typicode.com/users/?" - duration: 115ms -``` - -**Key issues**: -- ❌ No `aspnet_core.request` span exists in worker process -- ❌ Worker's `azure_functions.invoke` is parented to HOST's root span (1978509666546725896) -- ❌ Host creates unnecessary `azure_functions.invoke` and `http.request` spans -- ✓ All spans are in the same trace (distributed tracing working via headers) - -### Phase 2: After Enabling AspNetCoreDiagnosticObserver in Worker - -**Location**: [`2-after-enabling-aspnetcore-observer-in-worker-process/`](2-after-enabling-aspnetcore-observer-in-worker-process/) - -After enabling AspNetCoreDiagnosticObserver in the worker process, the `aspnet_core.request` span is now created, but the worker's `azure_functions.invoke` span has incorrect parenting. - -**Captured traces**: -- [trace_payload_1762824450957.json](2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824450957.json) - Worker spans -- [trace_payload_1762824451432.json](2-after-enabling-aspnetcore-observer-in-worker-process/trace_payload_1762824451432.json) - Host spans - -**Analysis of trace `11227614026327825028`**: -``` -Host spans: -├─ azure_functions.invoke (span: 12539548146409755932) [ROOT] ❌ Should not exist -└─ http.request (span: 17651134856053342758) [child of above] ❌ Should not exist - -Worker spans (all in same trace ✓): -├─ aspnet_core.request (span: 5312880851904873807) ✅ NOW EXISTS! -│ └─ parent: 17651134856053342758 (host's http.request) ✓ Correct! -└─ azure_functions.invoke (span: 727838728754785615) - └─ parent: 12539548146409755932 (host's root) ❌ Should be child of aspnet_core.request! -``` - -**Progress made**: -- ✅ `aspnet_core.request` span now exists in worker process -- ✅ `aspnet_core.request` correctly parented to host's `http.request` span -- ❌ Worker's `azure_functions.invoke` still parented to **host's root span** instead of `aspnet_core.request` -- ❌ Host still creates unnecessary `azure_functions.invoke` and `http.request` spans - -### Phase 3: Fixed HttpContext.Items Bridge ✅ RESOLVED - -**Commit**: `1d3179aa5` - Fix Azure Functions span parenting with ASP.NET Core - -**Root cause identified**: Looking for wrong key in `FunctionContext.Items` -- ❌ Old: `"__AspNetCoreHttpContext__"` (doesn't exist) -- ✅ Fixed: `"HttpRequestContext"` (actual key set by Azure Functions Worker SDK) - -**Changes made**: -1. Skip stale gRPC header extraction when ASP.NET Core integration detected (`AzureFunctionsCommon.cs:242-243`) -2. Use correct `"HttpRequestContext"` key for HttpContext lookup (`AzureFunctionsCommon.cs:292`) -3. Added comprehensive debug logging for troubleshooting - -**Analysis of trace `14656220060439490006`** (Nov 12, 2025): -``` -Worker spans: -├─ aspnet_core.request (span: 10174520177415259312) -│ └─ azure_functions.invoke (span: 1243553223469235235) ✅ CORRECT PARENT! -│ └─ test_span (span: 11352266471047046875) -│ └─ http.request (span: 3397667712105060208) -``` - -**Result**: -- ✅ Worker's `azure_functions.invoke` correctly parented to `aspnet_core.request` -- ✅ Proper span hierarchy in worker process -- ✅ All spans in same trace -- ⚠️ Host spans still present (secondary priority) - -## Code Changes Made - -### 1. Enable AspNetCoreDiagnosticObserver in Worker Process - -**File**: `tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs:473-493` - -Modified to enable `AspNetCoreDiagnosticObserver` in isolated worker process only: - -```csharp -var isInAzureFunctionsHost = EnvironmentHelpers.IsRunningInAzureFunctionsHost(); -var shouldSkipAspNetCore = isInAzureFunctionsHost || - (EnvironmentHelpers.IsAzureFunctions() && !EnvironmentHelpers.IsAzureFunctionsIsolated()); - -if (shouldSkipAspNetCore) -{ - Log.Debug("Skipping AspNetCoreDiagnosticObserver in Azure Functions (host or in-process)."); -} -else -{ - observers.Add(new AspNetCoreDiagnosticObserver()); -} -``` - -### 2. Add Process Identification Tag - -**File**: `tracer/src/Datadog.Trace/Agent/MessagePack/SpanMessagePackFormatter.cs:742-754` - -Added `aas.function.process` tag to distinguish host vs worker spans: -- Host spans: `aas.function.process: host` -- Worker spans: `aas.function.process: worker` - -## Testing Workflow - -### Environment - -**All Function Apps** (Resource Group: `lucas.pimentel`, Location: Canada Central): - -| Name | Purpose | -|------|---------| -| lucasp-premium-linux-isolated-aspnet | **Primary test app** - Isolated .NET 8 with ASP.NET Core Integration | -| lucasp-premium-linux-isolated | Isolated .NET 8 (no ASP.NET Core) | -| lucasp-premium-linux-inproc | In-process .NET 6 | -| lucasp-premium-windows-isolated-aspnet | Windows isolated with ASP.NET Core | -| lucasp-premium-windows-isolated | Windows isolated (no ASP.NET Core) | -| lucasp-premium-windows-inproc | Windows in-process | -| lucasp-consumption-windows-isolated | Windows consumption plan | -| lucasp-flex-consumption-isolated | Flex consumption plan | - -**Primary Test App:** -- **Name**: `lucasp-premium-linux-isolated-aspnet` -- **Resource Group**: `lucas.pimentel` -- **Source**: `D:\source\datadog\serverless-dev-apps\azure\functions\dotnet\isolated-dotnet8-aspnetcore` - -### Steps - -1. **Build NuGet package**: - ```powershell - .\tracer\tools\Build-AzureFunctionsNuget.ps1 -CopyTo D:\temp\nuget -Verbose - ``` - -2. **Deploy test app**: - ```bash - cd D:/source/datadog/serverless-dev-apps/azure/functions/dotnet/isolated-dotnet8-aspnetcore - dotnet restore - func azure functionapp publish lucasp-premium-linux-isolated - ``` - - Wait 1-2 minutes for worker restart after deployment. - -3. **Trigger function**: - ```bash - curl https://lucasp-premium-linux-isolated.azurewebsites.net/api/HttpTest - ``` - -4. **Download logs**: - ```bash - az functionapp log download \ - --name lucasp-premium-linux-isolated \ - --resource-group lucas.pimentel \ - --log-path D:/temp/logs-$(date +%H%M%S).zip - ``` - -5. **Query Datadog** (optional): - ```bash - # Filter by process type - curl -X POST https://api.datadoghq.com/api/v2/spans/events/search \ - -H "DD-API-KEY: ${DD_API_KEY}" \ - -H "DD-APPLICATION-KEY: ${DD_APPLICATION_KEY}" \ - -H "Content-Type: application/json" \ - -d '{"data":{"attributes":{"filter":{"query":"service:lucasp-premium-linux-isolated @aas.function.process:worker","from":"now-10m","to":"now"}},"type":"search_request"}}' - ``` - -### Verification ✅ - -**Expected trace structure** (after fix): -``` -Worker aspnet_core.request (ROOT) -└─ Worker azure_functions.invoke (child) ✓ - └─ test_span - └─ http.request -``` - -**Verified** (Trace ID: `14656220060439490006`, Nov 12 2025): -- ✅ Worker's `azure_functions.invoke` parent_id matches `aspnet_core.request` span_id -- ✅ All spans in same trace -- ✅ Proper span hierarchy in worker process -- ⚠️ Host spans still present (secondary priority to remove) - -## Minimal Reproduction - -```csharp -[Function(nameof(HttpTest))] -public async Task HttpTest([HttpTrigger(AuthorizationLevel.Anonymous, "get")] HttpRequest request) -{ - using (var scope = Tracer.Instance.StartActive("test_span")) - { - using var httpClient = new HttpClient(); - await httpClient.GetStringAsync("https://jsonplaceholder.typicode.com/users/1"); - return new OkObjectResult(new { message = "success" }); - } -} -``` - -**Requirements**: -- Isolated Azure Functions (.NET Isolated) -- ASP.NET Core Integration (`ConfigureFunctionsWebApplication()`) -- HTTP Trigger -- `Datadog.AzureFunctions` NuGet package - -## Goal - -Now that we have an `aspnet_core.request` span in the worker process (Phase 2), we need to: - -1. **Make worker's `azure_functions.invoke` span a child of `aspnet_core.request`** (high priority) - - Currently: Worker's `azure_functions.invoke` is parented to host's root span - - Goal: Worker's `azure_functions.invoke` should be parented to worker's `aspnet_core.request` span - - This will create the correct hierarchy: `aspnet_core.request` → `azure_functions.invoke` → user code - -2. **Remove host-side spans** (secondary priority) - - Host's `azure_functions.invoke` and `http.request` spans are unnecessary when ASP.NET Core integration is enabled - - These spans represent HTTP proxying overhead, not the actual function execution - - Detect HTTP proxying in host process and skip span creation - -## Implementation Plan - -### Approach: Use HttpContext.Items as Bridge - -Since AsyncLocal context doesn't flow through Azure Functions middleware, use `HttpContext.Items` to explicitly pass span context between middleware layers. - -**Step 1: Store scope in HttpContext.Items** -- File: `tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs:125` -- After creating scope, store in `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` -- Use `__` prefix to avoid conflicts with user code (same pattern as `TracingHttpModule.cs`) - -**Step 2: Retrieve scope in Azure Functions middleware** -- File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs:262-279` -- Get HttpContext from `functionContext.GetHttpContext()` -- Retrieve scope from `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` -- Use as parent before falling back to extraction logic - -**Step 3: Skip host span creation when proxying** -- File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsExecutorTryExecuteAsyncIntegration.cs` -- Detect `IsRunningInAzureFunctionsHost()` + HTTP proxying enabled -- Skip span creation in host when ASP.NET Core integration is active in worker - -**Why this works:** -- HttpContext.Items persists throughout request lifecycle -- Both ASP.NET Core and Azure Functions middleware access same HttpContext instance -- No reliance on AsyncLocal or Activity.Current -- Explicit and debuggable - -**Fallback approaches:** -- Use `FunctionContext.Features` to store/retrieve scope -- Custom middleware to bridge AsyncLocal → Features before context is lost - -## Implementation Progress - -### Phase 3: HttpContext.Items Bridge (In Progress) - -**Status**: Code implemented, testing pending - -**Changes Made** (Commit: d0e31854a): - -1. **Store scope in HttpContext.Items** ✅ - - File: `tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs:142` - - Added: `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"] = scope;` - - Stores the AspNetCore scope immediately after creation - -2. **Add Items property to IFunctionContext** ✅ - - File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs:22` - - Added: `IDictionary? Items { get; }` - - Allows duck-typed access to FunctionContext.Items - -3. **Retrieve scope from HttpContext.Items** ✅ - - File: `tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs:262-297` - - Logic: - 1. Check if `tracer.InternalActiveScope == null` (AsyncLocal didn't flow) - 2. Try to get HttpContext from `context.Items["__AspNetCoreHttpContext__"]` - 3. Try to get scope from `httpContext.Items["__Datadog.Trace.AspNetCore.ActiveScope"]` - 4. Use scope as parent if found, otherwise fall back to header extraction - - Only uses HttpContext.Items when AsyncLocal fails (maintains backward compatibility) - -**Next Steps**: -1. Deploy updated NuGet package to `lucasp-premium-linux-isolated-aspnet` -2. Trigger HttpTest function -3. Download and analyze traces to verify: - - Worker's `azure_functions.invoke` is now child of `aspnet_core.request` - - All spans in same trace - - Correct parent_id relationships - -**Pending**: -- Step 3: Skip host span creation when HTTP proxying is detected (secondary priority) - -## References - -- [Azure Functions ASP.NET Core Integration](https://learn.microsoft.com/en-us/azure/azure-functions/dotnet-isolated-process-guide?tabs=hostbuilder%2Cwindows#aspnet-core-integration) -- [Azure Functions Host source](https://github.com/Azure/azure-functions-host) -- [YARP (Yet Another Reverse Proxy)](https://microsoft.github.io/reverse-proxy/) From 7d855fcabf63bb30a802d001440c0c3dbd3724da Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 30 Jan 2026 11:21:32 -0500 Subject: [PATCH 10/42] update test snapshots --- ...AzureFunctionsTests.InProcess.verified.txt | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt index 78b237eb8db5..be62a8395185 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.InProcess.verified.txt @@ -13,11 +13,9 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Triggers.TriggerBindingSource`1[[Microsoft.Azure.WebJobs.TimerInfo, Microsoft.Azure.WebJobs.Extensions, Version=0.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35]], - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.TriggerAllTimer, aas.function.name: TriggerAllTimer, aas.function.trigger: Timer, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, @@ -77,15 +75,13 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Trigger, aas.function.name: TriggerCaller, aas.function.trigger: Http, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: aspnet_core, + component: AzureFunctions, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, @@ -260,15 +256,13 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.SimpleHttpTrigger, aas.function.name: SimpleHttpTrigger, aas.function.trigger: Http, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: aspnet_core, + component: AzureFunctions, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, @@ -301,15 +295,13 @@ aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.Exception, aas.function.name: Exception, aas.function.trigger: Http, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: aspnet_core, + component: AzureFunctions, env: integration_tests, error.msg: Exception while executing function: Exception, error.stack: @@ -348,15 +340,13 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequest req, ILo aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.ServerError, aas.function.name: ServerError, aas.function.trigger: Http, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: aspnet_core, + component: AzureFunctions, env: integration_tests, error.msg: The HTTP response has status code 500., http.method: GET, @@ -389,15 +379,13 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequest req, ILo aas.environment.os: unknown, aas.environment.runtime: .NET, aas.function.binding: Microsoft.Azure.WebJobs.Host.Executors.BindingSource, - aas.function.extension_version: ~4, aas.function.method: Samples.AzureFunctions.AllTriggers.AllTriggers.BadRequest, aas.function.name: BadRequest, aas.function.trigger: Http, - aas.function.worker_runtime: dotnet, aas.site.kind: functionapp, aas.site.name: AzureFunctionsAllTriggers, aas.site.type: function, - component: aspnet_core, + component: AzureFunctions, env: integration_tests, http.method: GET, http.request.headers.host: localhost:00000, From 2df5aa0853151a33f1b47d0677613b0730c29ca7 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Mon, 2 Feb 2026 18:15:49 -0500 Subject: [PATCH 11/42] fix span count in test --- .../AzureFunctionsTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs index ed50c2b78d43..65e3a52164bb 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs @@ -254,7 +254,7 @@ public async Task SubmitsTraces() await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore1"); - spans.Count.Should().Be(expectedSpanCount); + filteredSpans.Count.Should().Be(expectedSpanCount); } } } @@ -393,7 +393,7 @@ public async Task SubmitsTraces() await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore"); - spans.Count.Should().Be(expectedSpanCount); + filteredSpans.Count.Should().Be(expectedSpanCount); } } } From 9103661b66c99f8067a806932b41b2f965eb38c1 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 4 Feb 2026 14:36:29 -0500 Subject: [PATCH 12/42] remove unused `using` directives --- .../Azure/Functions/AzureFunctionsCommon.cs | 1 - .../PlatformHelpers/AspNetCoreHttpRequestHandler.cs | 4 ---- 2 files changed, 5 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 41650e9ad4c7..3dccb19be819 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -15,7 +15,6 @@ using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.Configuration; using Datadog.Trace.DuckTyping; -using Datadog.Trace.Headers; using Datadog.Trace.Logging; using Datadog.Trace.PlatformHelpers; using Datadog.Trace.Propagators; diff --git a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs index 8721756ce468..3658a006cba2 100644 --- a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs +++ b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs @@ -5,10 +5,8 @@ #if !NETFRAMEWORK using System; -using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; -using System.Linq; using Datadog.Trace.Activity; using Datadog.Trace.Activity.DuckTypes; using Datadog.Trace.Activity.Helpers; @@ -20,9 +18,7 @@ using Datadog.Trace.DataStreamsMonitoring.TransactionTracking; using Datadog.Trace.DiagnosticListeners; using Datadog.Trace.DuckTyping; -using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Headers; -using Datadog.Trace.Iast; using Datadog.Trace.Logging; using Datadog.Trace.Propagators; using Datadog.Trace.Tagging; From 658fbe8b3bc96272fd8fcc4c7ac42b60166da9bf Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 4 Feb 2026 14:37:13 -0500 Subject: [PATCH 13/42] use a const string, reorder fields --- .../Azure/Functions/AzureFunctionsCommon.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 3dccb19be819..babed4cc4300 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -30,15 +30,16 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions { internal static class AzureFunctionsCommon { - public const string IntegrationName = nameof(Configuration.IntegrationId.AzureFunctions); + private const string HttpRequestContextKey = "HttpRequestContext"; + private const string SpanType = SpanTypes.Serverless; + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AzureFunctionsCommon)); + public const string IntegrationName = nameof(Configuration.IntegrationId.AzureFunctions); public const string OperationName = AzureFunctionsConstants.AzureFunctionName; - public const string SpanType = SpanTypes.Serverless; public const string AzureApim = AzureFunctionsConstants.AzureApimName; public const IntegrationId IntegrationId = Configuration.IntegrationId.AzureFunctions; - private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AzureFunctionsCommon)); - public static CallTargetState OnFunctionExecutionBegin(TTarget instance, TFunction instanceParam) where TFunction : IFunctionInstance { @@ -256,11 +257,11 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even { case "Http": { - // Detect ASP.NET Core integration by checking for HttpContext in FunctionContext.Items - // In ASP.NET Core mode, HTTP requests are proxied directly (not via gRPC) - // The headers in the gRPC message are STALE (contain host's root span context) + // Detect ASP.NET Core integration by checking for HttpContext in FunctionContext.Items. + // In ASP.NET Core mode, HTTP requests are proxied directly (not via gRPC). + // The headers in the gRPC message are STALE (contain host's root span context). // The key "HttpRequestContext" is set by FunctionsHttpProxyingMiddleware in the worker - var isAspNetCoreIntegration = functionContext.Items?.ContainsKey("HttpRequestContext") == true; + var isAspNetCoreIntegration = functionContext.Items?.ContainsKey(HttpRequestContextKey) == true; if (isAspNetCoreIntegration) { @@ -358,7 +359,7 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even try { if (functionContext.Items != null && - functionContext.Items.TryGetValue("HttpRequestContext", out var httpContextObj) && + functionContext.Items.TryGetValue(HttpRequestContextKey, out var httpContextObj) && httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && httpContext.Items.TryGetValue(AspNetCoreHttpRequestHandler.HttpContextActiveScopeKey, out var scopeObj) && scopeObj is Scope aspNetCoreScope) From 8d2c38843ab4df456badbe043cf132642be680f3 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 4 Feb 2026 14:38:19 -0500 Subject: [PATCH 14/42] make CreateScope() private --- .../AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index babed4cc4300..391bbcfd8216 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -62,7 +62,7 @@ public static CallTargetState OnFunctionExecutionBegin(TTarg return new CallTargetState(scope); } - internal static Scope? CreateScope(Tracer tracer, TFunction instanceParam) + private static Scope? CreateScope(Tracer tracer, TFunction instanceParam) where TFunction : IFunctionInstance { Scope? scope = null; From e62eea2d81d0d144c0704d780c31af250bbea7eb Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 4 Feb 2026 15:05:52 -0500 Subject: [PATCH 15/42] small clean up / refactor --- .../Azure/Functions/AzureFunctionsCommon.cs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 391bbcfd8216..7a3bd4fea83b 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -290,23 +290,22 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even break; } - var functionName = functionContext.FunctionDefinition.Name; - var tags = new AzureFunctionsTags { TriggerType = triggerType, - ShortName = functionName, + ShortName = functionContext.FunctionDefinition.Name, FullName = functionContext.FunctionDefinition.EntryPoint, }; - // If active scope didn't flow via AsyncLocal, try to get it from HttpContext.Items - // (for HTTP triggers using ASP.NET Core integration). + // If active scope didn't flow via AsyncLocal, try to get it from HttpContext.Items (for HTTP triggers using ASP.NET Core integration). // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow. - var parentScope = tracer.InternalActiveScope ?? GetAspNetCoreScope(functionContext); + var parentSpanContext = tracer.InternalActiveScope?.Span.Context ?? + GetAspNetCoreScope(functionContext)?.Span.Context ?? + extractedContext.SpanContext; - if (parentScope == null) + if (parentSpanContext == null) { - // no local parent available, we are creating a local root span + // no local parent available, create a local root span tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); scope = tracer.StartActiveInternal(OperationName, parent: extractedContext.SpanContext, tags: tags); @@ -321,20 +320,21 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even } else { - scope = tracer.StartActiveInternal(OperationName, parent: parentScope.Span.Context, tags: tags); + scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); + var rootSpan = scope.Root.Span; // copy some tags to the root span - var rootSpan = scope.Root.Span; AzureFunctionsTags.SetRootSpanTags( rootSpan, - shortName: functionName, - fullName: functionContext.FunctionDefinition.EntryPoint, + shortName: tags.ShortName, + fullName: tags.FullName, bindingSource: rootSpan.Tags is AzureFunctionsTags t ? t.BindingSource : null, - triggerType: triggerType); + triggerType: tags.TriggerType); } scope.Root.Span.Type = SpanType; - scope.Span.ResourceName = $"{triggerType} {functionName}"; + + scope.Span.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; scope.Span.Type = SpanType; tracer.TracerManager.Telemetry.IntegrationGeneratedSpan(IntegrationId); } @@ -356,6 +356,7 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even // AsyncLocal context didn't flow - try to get parent scope from HttpContext.Items // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow Scope? parentScope = null; + try { if (functionContext.Items != null && From 8a55153e5e0b8297dfce59816b862dc707f1def7 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 4 Feb 2026 15:17:40 -0500 Subject: [PATCH 16/42] more small clean up / refactor --- .../Azure/Functions/AzureFunctionsCommon.cs | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 7a3bd4fea83b..bd7fff16c4bb 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -297,33 +297,25 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even FullName = functionContext.FunctionDefinition.EntryPoint, }; - // If active scope didn't flow via AsyncLocal, try to get it from HttpContext.Items (for HTTP triggers using ASP.NET Core integration). - // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow. + // Try to get parent span context from (in order): + // - existing local span + // - HttpContext.Items, for HTTP triggers using ASP.NET Core integration. Set in AspNetCoreHttpRequestHandler.StartAspNetCorePipelineScope(). + // - extracted from propagation headers var parentSpanContext = tracer.InternalActiveScope?.Span.Context ?? GetAspNetCoreScope(functionContext)?.Span.Context ?? extractedContext.SpanContext; - if (parentSpanContext == null) + scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); + var rootSpan = scope.Root.Span; + + if (scope.Span == rootSpan) { - // no local parent available, create a local root span + // this is the local root span tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); - scope = tracer.StartActiveInternal(OperationName, parent: extractedContext.SpanContext, tags: tags); - - if (extractedContext.SpanContext is { } extractedSpanContext) - { - Log.Debug("Azure Functions span creation: Parented to extracted context. parent_id: {ParentId}, trace_id: {TraceId}", extractedSpanContext.SpanId, extractedSpanContext.TraceId); - } - else - { - Log.Debug("Azure Functions span creation: Created as root span (no parent available)"); - } } else { - scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); - var rootSpan = scope.Root.Span; - - // copy some tags to the root span + // this is NOT the local root span, copy some tags to the root span AzureFunctionsTags.SetRootSpanTags( rootSpan, shortName: tags.ShortName, From af95bf7c91c817095d9ed712ceef8eccbb30a932 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 5 Feb 2026 13:31:46 -0500 Subject: [PATCH 17/42] small refactor --- .../Azure/Functions/AzureFunctionsCommon.cs | 17 +++++++------- .../Tagging/AzureFunctionsTags.cs | 23 +++++++++++++------ 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index bd7fff16c4bb..e565fa0fa17e 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -131,7 +131,7 @@ public static CallTargetState OnFunctionExecutionBegin(TTarg { // We don't want to create a new scope here when running isolated functions, // otherwise it is essentially a duplicate of the span created inside the - // isolated app, but we _do_ want to populate the "root" span here with the appropriate names + // isolated app. We _do_ want to populate the "root" span here with the appropriate names // and update it to be a "serverless" span. if (!isProxySpan) { @@ -173,7 +173,7 @@ public static CallTargetState OnFunctionExecutionBegin(TTarg if (!isProxySpan) { AzureFunctionsTags.SetRootSpanTags( - scope.Root.Span, + scope.Root.Span.Tags, shortName: functionName, fullName: instanceParam.FunctionDescriptor.FullName, bindingSource: bindingSourceType.FullName, @@ -306,9 +306,10 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even extractedContext.SpanContext; scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); + var span = scope.Span; var rootSpan = scope.Root.Span; - if (scope.Span == rootSpan) + if (span == rootSpan) { // this is the local root span tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); @@ -317,17 +318,17 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even { // this is NOT the local root span, copy some tags to the root span AzureFunctionsTags.SetRootSpanTags( - rootSpan, + rootSpan.Tags, shortName: tags.ShortName, fullName: tags.FullName, bindingSource: rootSpan.Tags is AzureFunctionsTags t ? t.BindingSource : null, triggerType: tags.TriggerType); - } - scope.Root.Span.Type = SpanType; + rootSpan.Type = SpanType; // "serverless" + } - scope.Span.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; - scope.Span.Type = SpanType; + span.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; + span.Type = SpanType; tracer.TracerManager.Telemetry.IntegrationGeneratedSpan(IntegrationId); } catch (Exception ex) diff --git a/tracer/src/Datadog.Trace/Tagging/AzureFunctionsTags.cs b/tracer/src/Datadog.Trace/Tagging/AzureFunctionsTags.cs index ca01f44f2a1e..ddfe1efda70b 100644 --- a/tracer/src/Datadog.Trace/Tagging/AzureFunctionsTags.cs +++ b/tracer/src/Datadog.Trace/Tagging/AzureFunctionsTags.cs @@ -34,14 +34,13 @@ internal sealed partial class AzureFunctionsTags : InstrumentationTags public string TriggerType { get; set; } = "Unknown"; internal static void SetRootSpanTags( - Span span, + ITags tags, string shortName, string fullName, string bindingSource, string triggerType) { - var tags = span.Tags; - if (span.Tags is AspNetCoreTags aspNetTags) + if (tags is AspNetCoreTags aspNetTags) { aspNetTags.InstrumentationName = ComponentName; } @@ -51,10 +50,20 @@ internal static void SetRootSpanTags( tags.SetTag(Tags.InstrumentationName, ComponentName); } - tags.SetTag(ShortNameTagName, shortName); - tags.SetTag(FullNameTagName, fullName); - tags.SetTag(BindingSourceTagName, bindingSource); - tags.SetTag(TriggerTypeTagName, triggerType); + if (tags is AzureFunctionsTags azureFunctionsTags) + { + azureFunctionsTags.ShortName = shortName; + azureFunctionsTags.FullName = fullName; + azureFunctionsTags.BindingSource = bindingSource; + azureFunctionsTags.TriggerType = triggerType; + } + else + { + tags.SetTag(ShortNameTagName, shortName); + tags.SetTag(FullNameTagName, fullName); + tags.SetTag(BindingSourceTagName, bindingSource); + tags.SetTag(TriggerTypeTagName, triggerType); + } } } } From 45b9849aca53077e287ab78e5e71dab288faff14 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 5 Feb 2026 17:49:59 -0500 Subject: [PATCH 18/42] fix tests --- .../AzureFunctionsTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs index 65e3a52164bb..87544a926167 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs @@ -247,6 +247,7 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 26; var spans = await agent.WaitForSpansAsync(expectedSpanCount); + spans.Count.Should().Be(expectedSpanCount); var filteredSpans = FilterOutSocketsHttpHandler(spans); @@ -254,7 +255,6 @@ public async Task SubmitsTraces() await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore1"); - filteredSpans.Count.Should().Be(expectedSpanCount); } } } @@ -378,6 +378,7 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 26; var spans = await agent.WaitForSpansAsync(expectedSpanCount); + spans.Should().HaveCount(expectedSpanCount); // There are _additional_ spans created for these compared to the non-AspNetCore version // These are http-client-handler-type: System.Net.Http.SocketsHttpHandler that come in around @@ -392,8 +393,6 @@ public async Task SubmitsTraces() using var s = new AssertionScope(); await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore"); - - filteredSpans.Count.Should().Be(expectedSpanCount); } } } From 95bb694c74648e5b2b720a576c38400c2afa4cc4 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 5 Feb 2026 17:58:55 -0500 Subject: [PATCH 19/42] clean up test code --- .../AzureFunctionsTests.cs | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs index 87544a926167..9a5506be8eec 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs @@ -145,15 +145,15 @@ public InProcessRuntimeV3(ITestOutputHelper output) public async Task SubmitsTraces() { using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); + using (await RunAzureFunctionAndWaitForExit(agent)) { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); + filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); - filteredSpans.Count.Should().Be(expectedSpanCount); - await AssertInProcessSpans(filteredSpans); } } @@ -179,14 +179,15 @@ public InProcessRuntimeV4(ITestOutputHelper output) public async Task SubmitsTraces() { using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true, useStatsD: true); + using (await RunAzureFunctionAndWaitForExit(agent, framework: "net6.0")) { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); + filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); - await AssertInProcessSpans(filteredSpans); } } @@ -213,13 +214,15 @@ public IsolatedRuntimeV4SdkV1(ITestOutputHelper output) public async Task SubmitsTraces() { using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); + using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); + using var s = new AssertionScope(); await AssertIsolatedSpans(filteredSpans, $"{nameof(AzureFunctionsTests)}.Isolated.V4.Sdk1"); } } @@ -247,14 +250,12 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 26; var spans = await agent.WaitForSpansAsync(expectedSpanCount); - spans.Count.Should().Be(expectedSpanCount); + spans.Should().HaveCount(expectedSpanCount); var filteredSpans = FilterOutSocketsHttpHandler(spans); using var s = new AssertionScope(); - await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore1"); - } } } @@ -283,21 +284,23 @@ public async Task SubmitsTraces() // so we will enable them with a lot of logging SetEnvironmentVariable("DD_LOGS_DIRECT_SUBMISSION_AZURE_FUNCTIONS_HOST_ENABLED", "true"); SetEnvironmentVariable("DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL", "VERBOSE"); - var hostName = "integration_ilogger_az_tests"; + const string hostName = "integration_ilogger_az_tests"; + using var logsIntake = new MockLogsIntake(); EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.ILogger), hostName); + using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); + using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); + filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); await AssertIsolatedSpans(filteredSpans); - filteredSpans.Count.Should().Be(expectedSpanCount); - var logs = logsIntake.Logs; // ~327 (ish) logs but we kill func.exe so some logs are lost @@ -331,20 +334,23 @@ public async Task SubmitsTraces() { SetEnvironmentVariable("DD_LOGS_DIRECT_SUBMISSION_AZURE_FUNCTIONS_HOST_ENABLED", "false"); SetEnvironmentVariable("DD_LOGS_DIRECT_SUBMISSION_MINIMUM_LEVEL", "VERBOSE"); - var hostName = "integration_ilogger_az_tests"; + const string hostName = "integration_ilogger_az_tests"; + using var logsIntake = new MockLogsIntake(); EnableDirectLogSubmission(logsIntake.Port, nameof(IntegrationId.ILogger), hostName); using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); + using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); + filteredSpans.Should().HaveCount(expectedSpanCount); + using var s = new AssertionScope(); await AssertIsolatedSpans(filteredSpans, filename: $"{nameof(AzureFunctionsTests)}.Isolated.V4.HostLogsDisabled"); - filteredSpans.Count.Should().Be(expectedSpanCount); var logs = logsIntake.Logs; // we expect some logs still from the worker process @@ -374,6 +380,7 @@ public IsolatedRuntimeV4AspNetCore(ITestOutputHelper output) public async Task SubmitsTraces() { using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); + using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { const int expectedSpanCount = 26; @@ -386,13 +393,12 @@ public async Task SubmitsTraces() // because of this they cause a lot of flake in the snapshots where they shift places // opting to just scrub them from the snapshots - we also don't think that the spans provide much // value so they may be removed from being traced. - var filteredSpans = FilterOutSocketsHttpHandler(spans); - - filteredSpans = filteredSpans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); + var filteredSpans = FilterOutSocketsHttpHandler(spans) + .Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)) + .ToImmutableList(); using var s = new AssertionScope(); - - await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore"); + await AssertIsolatedSpans(filteredSpans, $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore"); } } } From 7a99829055440afaf05e811718425b0eed90d3ba Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 5 Feb 2026 18:02:40 -0500 Subject: [PATCH 20/42] re-order const members --- .../Azure/Functions/AzureFunctionsCommon.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index e565fa0fa17e..12adba48b342 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -33,13 +33,13 @@ internal static class AzureFunctionsCommon private const string HttpRequestContextKey = "HttpRequestContext"; private const string SpanType = SpanTypes.Serverless; - private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AzureFunctionsCommon)); - public const string IntegrationName = nameof(Configuration.IntegrationId.AzureFunctions); public const string OperationName = AzureFunctionsConstants.AzureFunctionName; public const string AzureApim = AzureFunctionsConstants.AzureApimName; public const IntegrationId IntegrationId = Configuration.IntegrationId.AzureFunctions; + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(AzureFunctionsCommon)); + public static CallTargetState OnFunctionExecutionBegin(TTarget instance, TFunction instanceParam) where TFunction : IFunctionInstance { From 23f9cd9163e5c46dfcd81ef0a8589811c4343e3d Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 5 Feb 2026 18:04:54 -0500 Subject: [PATCH 21/42] clarify comment --- tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index c1681b67a466..9ec084dc8fb5 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -537,7 +537,7 @@ private static bool SkipAspNetCoreDiagnosticObserver() if (!AzureInfo.Instance.IsAzureFunction) { - // we only need to skip in some Azure Functions + // we only ever skip AspNetCoreDiagnosticObserver in Azure Functions return false; } @@ -548,6 +548,7 @@ private static bool SkipAspNetCoreDiagnosticObserver() return true; } + // FUNCTIONS_WORKER_RUNTIME == "dotnet-isolated" if (!AzureInfo.Instance.IsIsolatedFunction) { // Skip AspNetCoreDiagnosticObserver in in-process Azure Functions From 241d59cdea975835122d5b4382a35751fc168eef Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 6 Feb 2026 11:30:58 -0500 Subject: [PATCH 22/42] clean up test code --- .../AzureFunctionsTests.cs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs index 9a5506be8eec..73f08a8766b9 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs @@ -151,9 +151,9 @@ public async Task SubmitsTraces() const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); await AssertInProcessSpans(filteredSpans); } } @@ -185,9 +185,9 @@ public async Task SubmitsTraces() const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); await AssertInProcessSpans(filteredSpans); } } @@ -220,9 +220,9 @@ public async Task SubmitsTraces() const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); await AssertIsolatedSpans(filteredSpans, $"{nameof(AzureFunctionsTests)}.Isolated.V4.Sdk1"); } } @@ -250,11 +250,11 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 26; var spans = await agent.WaitForSpansAsync(expectedSpanCount); - spans.Should().HaveCount(expectedSpanCount); var filteredSpans = FilterOutSocketsHttpHandler(spans); using var s = new AssertionScope(); + spans.Should().HaveCount(expectedSpanCount); await AssertIsolatedSpans(filteredSpans.ToImmutableList(), $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore1"); } } @@ -296,17 +296,15 @@ public async Task SubmitsTraces() const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); await AssertIsolatedSpans(filteredSpans); - var logs = logsIntake.Logs; - // ~327 (ish) logs but we kill func.exe so some logs are lost // and since sometimes the batch of logs can be 100+ it can be a LOT of logs that we lose // so just check that we have much more than when we have host logs disabled - logs.Should().HaveCountGreaterThanOrEqualTo(200); + logsIntake.Logs.Should().HaveCountGreaterThanOrEqualTo(200); } } } @@ -345,17 +343,16 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 21; var spans = await agent.WaitForSpansAsync(expectedSpanCount); - var filteredSpans = spans.Where(s => !s.Resource.Equals("Timer ExitApp", StringComparison.OrdinalIgnoreCase)).ToImmutableList(); - filteredSpans.Should().HaveCount(expectedSpanCount); using var s = new AssertionScope(); + filteredSpans.Should().HaveCount(expectedSpanCount); await AssertIsolatedSpans(filteredSpans, filename: $"{nameof(AzureFunctionsTests)}.Isolated.V4.HostLogsDisabled"); - var logs = logsIntake.Logs; // we expect some logs still from the worker process // this just seems flaky I THINK because of killing the func.exe process (even though we aren't using the host logs) // commonly see 13, 14, 15, 16 logs, but IF we were logging the host logs we'd see 300+ + var logs = logsIntake.Logs; logs.Should().HaveCountGreaterThan(10); logs.Should().HaveCountLessThanOrEqualTo(20); } @@ -385,7 +382,6 @@ public async Task SubmitsTraces() { const int expectedSpanCount = 26; var spans = await agent.WaitForSpansAsync(expectedSpanCount); - spans.Should().HaveCount(expectedSpanCount); // There are _additional_ spans created for these compared to the non-AspNetCore version // These are http-client-handler-type: System.Net.Http.SocketsHttpHandler that come in around @@ -398,6 +394,7 @@ public async Task SubmitsTraces() .ToImmutableList(); using var s = new AssertionScope(); + spans.Should().HaveCount(expectedSpanCount); await AssertIsolatedSpans(filteredSpans, $"{nameof(AzureFunctionsTests)}.Isolated.V4.AspNetCore"); } } From b583c084553074cf55e88285a6ec843c54149d9c Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 10 Feb 2026 15:33:07 -0500 Subject: [PATCH 23/42] fix parent context priority in isolated worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracted context from gRPC propagation headers must take priority over InternalActiveScope. Enabling the AspNetCoreDiagnosticObserver in isolated workers caused InternalActiveScope to be a gRPC listener span with an unrelated trace ID, breaking host-to-worker context flow. 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 12adba48b342..e830822ec1fd 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -298,12 +298,17 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even }; // Try to get parent span context from (in order): - // - existing local span - // - HttpContext.Items, for HTTP triggers using ASP.NET Core integration. Set in AspNetCoreHttpRequestHandler.StartAspNetCorePipelineScope(). - // - extracted from propagation headers - var parentSpanContext = tracer.InternalActiveScope?.Span.Context ?? + // 1. Extracted from propagation headers (gRPC message from the host process). + // This is the most reliable source of the host's trace context for non-ASP.NET Core apps. + // 2. HttpContext.Items bridge, for HTTP triggers using ASP.NET Core integration. + // In ASP.NET Core mode, gRPC headers are stale so we skip extraction above and + // retrieve the scope stored by AspNetCoreHttpRequestHandler instead. + // 3. Existing local span (fallback). + // Note: InternalActiveScope may be an unrelated aspnet_core.request span from the + // gRPC listener, so it should only be used as a last resort. + var parentSpanContext = extractedContext.SpanContext ?? GetAspNetCoreScope(functionContext)?.Span.Context ?? - extractedContext.SpanContext; + tracer.InternalActiveScope?.Span.Context; scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); var span = scope.Span; From b552197be92b36565c1c986adee6050debeef6c9 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 13 Feb 2026 13:39:39 -0500 Subject: [PATCH 24/42] Fix duplicate span creation in Azure Functions ASP.NET Core mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When ASP.NET Core integration is active, check if the retrieved scope is already the active scope before creating a new span. If it's active, reuse it and update the root span tags instead of creating a duplicate azure_functions.invoke span. This prevents extra spans that break integration tests expecting specific span counts. 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 77 ++++++++++++------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index e830822ec1fd..e26446fa8128 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -297,31 +297,22 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even FullName = functionContext.FunctionDefinition.EntryPoint, }; - // Try to get parent span context from (in order): - // 1. Extracted from propagation headers (gRPC message from the host process). - // This is the most reliable source of the host's trace context for non-ASP.NET Core apps. - // 2. HttpContext.Items bridge, for HTTP triggers using ASP.NET Core integration. - // In ASP.NET Core mode, gRPC headers are stale so we skip extraction above and - // retrieve the scope stored by AspNetCoreHttpRequestHandler instead. - // 3. Existing local span (fallback). - // Note: InternalActiveScope may be an unrelated aspnet_core.request span from the - // gRPC listener, so it should only be used as a last resort. - var parentSpanContext = extractedContext.SpanContext ?? - GetAspNetCoreScope(functionContext)?.Span.Context ?? - tracer.InternalActiveScope?.Span.Context; - - scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); - var span = scope.Span; - var rootSpan = scope.Root.Span; - - if (span == rootSpan) + // Try to get parent scope from (in order): + // 1. HttpContext.Items bridge, for HTTP triggers using ASP.NET Core integration. + // 2. Existing local span (fallback). + var aspNetCoreScope = GetAspNetCoreScope(functionContext); + var activeScope = tracer.InternalActiveScope; + + // Check if the ASP.NET Core scope is already active + if (aspNetCoreScope != null && activeScope == aspNetCoreScope) { - // this is the local root span - tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); - } - else - { - // this is NOT the local root span, copy some tags to the root span + // The ASP.NET Core span is already active - don't create a new span, + // just update the existing root span's tags to make it a "serverless" span. + // This matches the behavior for isolated worker scenario where we detect + // an existing scope and update it rather than creating a duplicate. + scope = activeScope; + var rootSpan = scope.Root.Span; + AzureFunctionsTags.SetRootSpanTags( rootSpan.Tags, shortName: tags.ShortName, @@ -330,10 +321,44 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even triggerType: tags.TriggerType); rootSpan.Type = SpanType; // "serverless" + rootSpan.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; + } + else + { + // Create a new span with the appropriate parent context from: + // 1. Extracted from propagation headers (gRPC message from the host process). + // 2. ASP.NET Core scope (if available but not active - shouldn't happen). + // 3. Existing local span (fallback). + var parentSpanContext = extractedContext.SpanContext ?? + aspNetCoreScope?.Span.Context ?? + activeScope?.Span.Context; + + scope = tracer.StartActiveInternal(OperationName, parent: parentSpanContext, tags: tags); + var span = scope.Span; + var rootSpan = scope.Root.Span; + + if (span == rootSpan) + { + // this is the local root span + tags.SetAnalyticsSampleRate(IntegrationId, tracer.CurrentTraceSettings.Settings, enabledWithGlobalSetting: false); + } + else + { + // this is NOT the local root span, copy some tags to the root span + AzureFunctionsTags.SetRootSpanTags( + rootSpan.Tags, + shortName: tags.ShortName, + fullName: tags.FullName, + bindingSource: rootSpan.Tags is AzureFunctionsTags t ? t.BindingSource : null, + triggerType: tags.TriggerType); + + rootSpan.Type = SpanType; // "serverless" + } + + span.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; + span.Type = SpanType; } - span.ResourceName = $"{tags.TriggerType} {tags.ShortName}"; - span.Type = SpanType; tracer.TracerManager.Telemetry.IntegrationGeneratedSpan(IntegrationId); } catch (Exception ex) From 2f833f682161a0b31b74dc679969d0bea025aed5 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 18 Feb 2026 18:34:23 -0500 Subject: [PATCH 25/42] rename parameter --- .../Azure/Functions/AzureFunctionsCommon.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index e26446fa8128..ceefdd9d1ad2 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -413,7 +413,7 @@ httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && return parentScope; } - private static PropagationContext ExtractPropagatedContextFromHttp(T context, string? bindingName) + private static PropagationContext ExtractPropagatedContextFromHttp(T functionContext, string? bindingName) where T : IFunctionContext { // Need to try and grab the headers from the context @@ -421,7 +421,7 @@ private static PropagationContext ExtractPropagatedContextFromHttp(T context, // directly from the grpc call instead. This is... interesting. It // is effectively doing the equivalent of context.GetHttpRequestDataAsync() which is // the suggested approach in the docs. - if (context.Features is null || string.IsNullOrEmpty(bindingName)) + if (functionContext.Features is null || string.IsNullOrEmpty(bindingName)) { return default; } @@ -429,7 +429,7 @@ private static PropagationContext ExtractPropagatedContextFromHttp(T context, try { object? feature = null; - foreach (var keyValuePair in context.Features) + foreach (var keyValuePair in functionContext.Features) { if (keyValuePair.Key.FullName?.Equals("Microsoft.Azure.Functions.Worker.Context.Features.IFunctionBindingsFeature") == true) { From 8663beae7e92d68e657b02cee55416bf18a37a95 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 18 Feb 2026 18:46:38 -0500 Subject: [PATCH 26/42] Reorder Azure Functions observer skip logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Co-Authored-By: Claude Code --- .../ClrProfiler/Instrumentation.cs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs index 9ec084dc8fb5..b91abeae913d 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/Instrumentation.cs @@ -85,7 +85,7 @@ private static void PropagateStableConfiguration() var tracerSettings = tracer.Settings; var mutableSettings = tracerSettings.Manager.InitialMutableSettings; - NativeInterop.SharedConfig config = new NativeInterop.SharedConfig + var config = new NativeInterop.SharedConfig { ProfilingEnabled = profilerSettings.ProfilerState switch { @@ -537,17 +537,11 @@ private static bool SkipAspNetCoreDiagnosticObserver() if (!AzureInfo.Instance.IsAzureFunction) { - // we only ever skip AspNetCoreDiagnosticObserver in Azure Functions + // We only skip AspNetCoreDiagnosticObserver in Azure Functions. + // Don't skip it outside Azure Functions. return false; } - if (AzureInfo.Instance.IsIsolatedFunctionHostProcess) - { - // Skip AspNetCoreDiagnosticObserver in Azure Functions _host_ processes - Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in an isolated Azure Function host process."); - return true; - } - // FUNCTIONS_WORKER_RUNTIME == "dotnet-isolated" if (!AzureInfo.Instance.IsIsolatedFunction) { @@ -556,12 +550,20 @@ private static bool SkipAspNetCoreDiagnosticObserver() return true; } + if (AzureInfo.Instance.IsIsolatedFunctionHostProcess) + { + // Skip AspNetCoreDiagnosticObserver in Azure Functions _host_ processes + Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in an isolated Azure Function host process."); + return true; + } + // FUNCTIONS_EXTENSION_VERSION var azureFunctionsExtensionVersion = AzureInfo.Instance.AzureFunctionsExtensionVersion; if (azureFunctionsExtensionVersion != "~4") { - // Skip AspNetCoreDiagnosticObserver in v1 functions (v2 and v3 are not supported at all) + // Skip AspNetCoreDiagnosticObserver in v1 isolated functions (v2 and v3 are not supported at all) + // to keep the previous behavior Log.Debug("Skipping AspNetCoreDiagnosticObserver: running in Azure Function with extension version {AzureFunctionsExtensionVersion}.", azureFunctionsExtensionVersion); return true; } From e886f0d7a4e5b30ee13ef7e73117c8a63e47a69b Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 19 Feb 2026 16:38:09 -0500 Subject: [PATCH 27/42] Use duck typing for HttpContext in GetAspNetCoreScope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace direct Microsoft.AspNetCore.Http.HttpContext type reference with IHttpContextItems duck type to avoid FileNotFoundException in non-ASP.NET Core Azure Functions workers where the Microsoft.AspNetCore.Http.Abstractions assembly is not available. 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 2 +- .../Functions/Isolated/IHttpContextItems.cs | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index ceefdd9d1ad2..043099a0cd36 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -384,7 +384,7 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even { if (functionContext.Items != null && functionContext.Items.TryGetValue(HttpRequestContextKey, out var httpContextObj) && - httpContextObj is Microsoft.AspNetCore.Http.HttpContext httpContext && + httpContextObj.TryDuckCast(out var httpContext) && httpContext.Items.TryGetValue(AspNetCoreHttpRequestHandler.HttpContextActiveScopeKey, out var scopeObj) && scopeObj is Scope aspNetCoreScope) { diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs new file mode 100644 index 000000000000..fc5d0b0e50a3 --- /dev/null +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs @@ -0,0 +1,23 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if !NETFRAMEWORK +#nullable enable + +using System.Collections.Generic; + +namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions; + +/// +/// Duck type for Microsoft.AspNetCore.Http.HttpContext, +/// used to avoid a hard assembly reference to Microsoft.AspNetCore.Http.Abstractions +/// which may not be available in non-ASP.NET Core Azure Functions workers. +/// +internal interface IHttpContextItems +{ + IDictionary Items { get; } +} + +#endif From 3685e45914ddc4694c30c8d0347102e1cd010aea Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 19 Feb 2026 20:39:15 -0500 Subject: [PATCH 28/42] update expected span counts --- .../AzureFunctionsTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs index 73f08a8766b9..989c95ecfed1 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AzureFunctionsTests.cs @@ -248,7 +248,7 @@ public async Task SubmitsTraces() using var agent = EnvironmentHelper.GetMockAgent(useTelemetry: true); using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { - const int expectedSpanCount = 26; + const int expectedSpanCount = 31; var spans = await agent.WaitForSpansAsync(expectedSpanCount); var filteredSpans = FilterOutSocketsHttpHandler(spans); @@ -380,7 +380,7 @@ public async Task SubmitsTraces() using (await RunAzureFunctionAndWaitForExit(agent, expectedExitCode: -1)) { - const int expectedSpanCount = 26; + const int expectedSpanCount = 31; var spans = await agent.WaitForSpansAsync(expectedSpanCount); // There are _additional_ spans created for these compared to the non-AspNetCore version From 085be2072ca57ffc5b8593902b9e9dcba2d48b1e Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 18 Mar 2026 17:28:36 -0400 Subject: [PATCH 29/42] fix bad merge --- .../AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 043099a0cd36..bf1314c9c0cd 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -141,7 +141,7 @@ public static CallTargetState OnFunctionExecutionBegin(TTarg ? functionName.Substring(10) : functionName; AzureFunctionsTags.SetRootSpanTags( - rootSpan, + rootSpan.Tags, shortName: remoteFunctionName, fullName: rootSpan.Tags is AzureFunctionsTags t ? t.FullName : null, // can't get anything meaningful here, so leave it as-is bindingSource: bindingSourceType.FullName, From d32aa62406caeec4275ff935001c170f8daa279f Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 19 Mar 2026 11:45:10 -0400 Subject: [PATCH 30/42] Update Azure Functions snapshots for new _dd.svc_src tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Co-Authored-By: Claude Code --- .../AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt | 3 ++- .../AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt index 554cde39b3e5..a622ccc624fd 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt @@ -688,7 +688,8 @@ at Samples.AzureFunctions.V4Isolated.AspNetCore.DirectFunctionExecutor.ExecuteAs out.host: localhost, runtime-id: Guid_1, span.kind: client, - _dd.base_service: AzureFunctionsAllTriggers + _dd.base_service: AzureFunctionsAllTriggers, + _dd.svc_src: http-client }, Metrics: { _dd.top_level: 1.0 diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt index aabaf13c7a35..e98ebf770d2d 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt @@ -687,7 +687,8 @@ at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req) out.host: localhost, runtime-id: Guid_1, span.kind: client, - _dd.base_service: AzureFunctionsAllTriggers + _dd.base_service: AzureFunctionsAllTriggers, + _dd.svc_src: http-client }, Metrics: { _dd.top_level: 1.0 From 18ada3a09f5a5d69cf262d5be53291094ace7e8a Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Fri, 10 Apr 2026 17:56:52 -0400 Subject: [PATCH 31/42] fix merge conflict --- .../PlatformHelpers/AspNetCoreHttpRequestHandler.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs index 3658a006cba2..4cd518cc3c90 100644 --- a/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs +++ b/tracer/src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs @@ -21,6 +21,7 @@ using Datadog.Trace.Headers; using Datadog.Trace.Logging; using Datadog.Trace.Propagators; +using Datadog.Trace.Serverless; using Datadog.Trace.Tagging; using Datadog.Trace.Util; using Datadog.Trace.Util.Http; @@ -172,7 +173,7 @@ private Scope StartAspNetCorePipelineScope(Tracer tracer, Security security, Ias httpContext.Items[HttpContextTrackingKey] = new RequestTrackingFeature(originalPath, scope, proxyContext?.Scope); #endif - if (EnvironmentHelpers.IsAzureFunctions()) + if (AzureInfo.Instance.IsAzureFunction) { // Store scope in HttpContext.Items for Azure Functions middleware to retrieve httpContext.Items[HttpContextActiveScopeKey] = scope; From 773b239506e9ac99fed75214d139f76c13485bba Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 14 Apr 2026 18:01:47 -0400 Subject: [PATCH 32/42] Add comment explaining stale gRPC headers in HTTP proxying mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index bf1314c9c0cd..2c189d4dfeb1 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -265,7 +265,10 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even if (isAspNetCoreIntegration) { - // Skip extraction - will rely on HttpContext.Items bridge or create root span + // Skip gRPC header extraction in HTTP proxying mode. + // The gRPC message headers contain STALE context from the host process + // (the host's root span), so extracting them would create an incorrect parent. + // Instead, we rely on the HttpContext.Items bridge to get the ASP.NET Core scope. Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected (HTTP proxying mode)"); } else From 17707c6086e62c95d9b5a43e2d0416b040debdc0 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 14 Apr 2026 18:12:51 -0400 Subject: [PATCH 33/42] Don't return borrowed ASP.NET Core scope from calltarget state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the ASP.NET Core scope is already active, we update its tags but no longer assign it to `scope`. This prevents OnAsyncMethodEnd from disposing a scope it doesn't own. 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 2c189d4dfeb1..d298e2b7f38a 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -311,10 +311,9 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even { // The ASP.NET Core span is already active - don't create a new span, // just update the existing root span's tags to make it a "serverless" span. - // This matches the behavior for isolated worker scenario where we detect - // an existing scope and update it rather than creating a duplicate. - scope = activeScope; - var rootSpan = scope.Root.Span; + // Don't assign to `scope`: the ASP.NET Core middleware owns this scope's + // lifetime, and returning it here would cause OnAsyncMethodEnd to dispose it. + var rootSpan = activeScope.Root.Span; AzureFunctionsTags.SetRootSpanTags( rootSpan.Tags, From 276f842a721e2f9ad0c106fc1e4e7bce5aa3b5f4 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 14 Apr 2026 18:33:09 -0400 Subject: [PATCH 34/42] Change scope retrieval error log from Debug to Error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An exception during duck typing / scope retrieval from HttpContext.Items is unexpected and should be logged at Error level. 🤖 Co-Authored-By: Claude Code --- .../AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index d298e2b7f38a..93f0f8b00ca6 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -409,7 +409,7 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even } catch (Exception ex) { - Log.Debug(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); + Log.Error(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); } return parentScope; From a1e54afed77fffe21518323cbaaceedd63498d6f Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 21 Apr 2026 16:44:35 -0400 Subject: [PATCH 35/42] Mark Items dictionary values as nullable in duck types --- .../Azure/Functions/Isolated/IFunctionContext.cs | 2 +- .../Azure/Functions/Isolated/IHttpContextItems.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs index 91ed96e88b56..24d23b8369e2 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IFunctionContext.cs @@ -19,7 +19,7 @@ internal interface IFunctionContext IEnumerable>? Features { get; } - IDictionary? Items { get; } + IDictionary? Items { get; } } #endif diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs index fc5d0b0e50a3..61e76253c2e0 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IHttpContextItems.cs @@ -17,7 +17,7 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions; /// internal interface IHttpContextItems { - IDictionary Items { get; } + IDictionary Items { get; } } #endif From f99d8434fb87598a2b85cc351e9a42fdf78a3c35 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 21 Apr 2026 16:54:58 -0400 Subject: [PATCH 36/42] Document source of HttpRequestContext key --- .../Azure/Functions/AzureFunctionsCommon.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 93f0f8b00ca6..3f449851839d 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -30,6 +30,11 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions { internal static class AzureFunctionsCommon { + // Key used by the Azure Functions .NET Worker to store the ASP.NET Core HttpContext in FunctionContext.Items. + // Defined in Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore: + // https://github.com/Azure/azure-functions-dotnet-worker/blob/596a94ef97f458d68381678fec6d92585e80d83d/extensions/Worker.Extensions.Http.AspNetCore/src/Constants.cs#L14 + // Set by FunctionsHttpProxyingMiddleware: + // https://github.com/Azure/azure-functions-dotnet-worker/blob/596a94ef97f458d68381678fec6d92585e80d83d/extensions/Worker.Extensions.Http.AspNetCore/src/FunctionsMiddleware/FunctionsHttpProxyingMiddleware.cs#L124 private const string HttpRequestContextKey = "HttpRequestContext"; private const string SpanType = SpanTypes.Serverless; From 7b1e57b08162e75cef3272abd3b48dbde4bf0a6b Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 21 Apr 2026 17:07:38 -0400 Subject: [PATCH 37/42] Add explicit null checks in ASP.NET Core scope retrieval --- .../Azure/Functions/AzureFunctionsCommon.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 3f449851839d..3fdf2a700aee 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -389,9 +389,11 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even try { - if (functionContext.Items != null && + if (functionContext.Items is not null && functionContext.Items.TryGetValue(HttpRequestContextKey, out var httpContextObj) && + httpContextObj is not null && httpContextObj.TryDuckCast(out var httpContext) && + httpContext is not null && httpContext.Items.TryGetValue(AspNetCoreHttpRequestHandler.HttpContextActiveScopeKey, out var scopeObj) && scopeObj is Scope aspNetCoreScope) { From eaa9b885c7d3b1a6e8c3d3b72d48645be825904f Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Tue, 21 Apr 2026 18:33:32 -0400 Subject: [PATCH 38/42] Set status 500 on aspnet_core.request when isolated worker function throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker's FunctionExecutionMiddleware catches user-function exceptions internally, so the ASP.NET Core diagnostic observer never sees an unhandled exception and HttpContext.Response.StatusCode stays at 200. Propagate the exception onto the aspnet_core.request span explicitly. 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommon.cs | 71 ++++++++++++++----- ...ionExecutionMiddlewareInvokeIntegration.cs | 7 ++ 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 3fdf2a700aee..d3ba44774f5b 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -15,6 +15,7 @@ using Datadog.Trace.ClrProfiler.CallTarget; using Datadog.Trace.Configuration; using Datadog.Trace.DuckTyping; +using Datadog.Trace.ExtensionMethods; using Datadog.Trace.Logging; using Datadog.Trace.PlatformHelpers; using Datadog.Trace.Propagators; @@ -210,20 +211,27 @@ public static CallTargetState OnIsolatedFunctionBegin(T functionContext) { var tracer = Tracer.Instance; - if (tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(IntegrationId)) + if (!tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(IntegrationId)) { - var scope = CreateIsolatedFunctionScope(tracer, functionContext); + return CallTargetState.GetDefault(); + } - if (scope != null) - { - return new CallTargetState(scope); - } + // Look up the aspnet_core.request scope once so we can both use it as parent + // (in CreateIsolatedFunctionScope) and propagate exceptions onto it later + // (in OnAsyncMethodEnd of the calling integration). + var aspNetCoreScope = GetAspNetCoreScope(functionContext); + + var scope = CreateIsolatedFunctionScope(tracer, functionContext, aspNetCoreScope); + + if (scope == null && aspNetCoreScope == null) + { + return CallTargetState.GetDefault(); } - return CallTargetState.GetDefault(); + return new CallTargetState(scope, state: aspNetCoreScope); } - private static Scope? CreateIsolatedFunctionScope(Tracer tracer, T functionContext) + private static Scope? CreateIsolatedFunctionScope(Tracer tracer, T functionContext, Scope? aspNetCoreScope) where T : IFunctionContext { Scope? scope = null; @@ -305,10 +313,8 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even FullName = functionContext.FunctionDefinition.EntryPoint, }; - // Try to get parent scope from (in order): - // 1. HttpContext.Items bridge, for HTTP triggers using ASP.NET Core integration. - // 2. Existing local span (fallback). - var aspNetCoreScope = GetAspNetCoreScope(functionContext); + // Use the supplied aspnet_core.request scope (from HttpContext.Items bridge) + // if available, otherwise fall back to the existing local active scope. var activeScope = tracer.InternalActiveScope; // Check if the ASP.NET Core scope is already active @@ -381,10 +387,9 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even private static Scope? GetAspNetCoreScope(T functionContext) where T : IFunctionContext { - Log.Debug("Azure Functions span creation: AsyncLocal context not available - attempting HttpContext.Items bridge"); - - // AsyncLocal context didn't flow - try to get parent scope from HttpContext.Items - // This happens in Azure Functions isolated worker where middleware breaks AsyncLocal flow + // Try to retrieve the aspnet_core.request scope via the HttpContext.Items bridge. + // This is needed because AsyncLocal context doesn't reliably flow between the worker's + // ASP.NET Core pipeline and the Azure Functions worker middleware pipeline. Scope? parentScope = null; try @@ -404,24 +409,52 @@ httpContext is not null && var spanContext = parentScope.Span.Context; Log.Debug( - "Azure Functions span creation: Retrieved AspNetCore scope {TraceId}-{SpanId}", + "Azure Functions: retrieved AspNetCore scope from HttpContext.Items {TraceId}-{SpanId}", spanContext.RawTraceId, spanContext.RawSpanId); } } else { - Log.Debug("Azure Functions span creation: Could not retrieve AspNetCore scope from HttpContext.Items"); + Log.Debug("Azure Functions: could not retrieve AspNetCore scope from HttpContext.Items"); } } catch (Exception ex) { - Log.Error(ex, "Azure Functions span creation: Error retrieving AspNetCore scope from HttpContext.Items"); + Log.Error(ex, "Azure Functions: error retrieving AspNetCore scope from HttpContext.Items"); } return parentScope; } + /// + /// Propagates an exception thrown by an isolated worker user function onto the + /// aspnet_core.request span. The exception is caught internally by the worker's + /// FunctionExecutionMiddleware, so the ASP.NET Core diagnostic observer never sees it + /// and the worker's HttpContext.Response.StatusCode remains at the default 200. Mirror + /// what AspNetCoreHttpRequestHandler.HandleAspNetCoreException does, without the AppSec + /// side effects (which already ran when the request entered ASP.NET Core). + /// + public static void SetExceptionOnAspNetCoreScope(Scope aspNetCoreScope, Exception exception, Tracer tracer) + { + try + { + var span = aspNetCoreScope.Span; + var settings = tracer.CurrentTraceSettings.Settings; + + if (!span.HasHttpStatusCode()) + { + span.SetHttpStatusCode(statusCode: 500, isServer: true, settings); + } + + span.SetException(exception); + } + catch (Exception ex) + { + Log.Error(ex, "Azure Functions: error propagating user function exception to AspNetCore scope"); + } + } + private static PropagationContext ExtractPropagatedContextFromHttp(T functionContext, string? bindingName) where T : IFunctionContext { diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/FunctionExecutionMiddlewareInvokeIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/FunctionExecutionMiddlewareInvokeIntegration.cs index 9aaf899bc4c5..d2ad6030355b 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/FunctionExecutionMiddlewareInvokeIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/FunctionExecutionMiddlewareInvokeIntegration.cs @@ -45,6 +45,13 @@ internal static CallTargetState OnMethodBegin(TTarget [PreserveContext] internal static TReturn OnAsyncMethodEnd(TTarget instance, TReturn returnValue, Exception exception, in CallTargetState state) { + // The worker's FunctionExecutionMiddleware catches this exception internally, + // so the aspnet_core.request span otherwise records status 200. Annotate it here. + if (exception is not null && state.State is Scope aspNetCoreScope) + { + AzureFunctionsCommon.SetExceptionOnAspNetCoreScope(aspNetCoreScope, exception, Tracer.Instance); + } + state.Scope?.DisposeWithException(exception); return returnValue; } From f5672dbcfb36f6a461e3ec70f53f40ff3173d831 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 22 Apr 2026 11:13:55 -0400 Subject: [PATCH 39/42] Fix CS8613 nullability mismatch in MockFunctionContext.Items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Co-Authored-By: Claude Code --- .../Azure/Functions/AzureFunctionsCommonTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs b/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs index e60e6dd7879c..452a76443567 100644 --- a/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs +++ b/tracer/test/Datadog.Trace.Tests/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommonTests.cs @@ -123,7 +123,7 @@ private class MockFunctionContext : IFunctionContext public IEnumerable>? Features { get; set; } - public IDictionary? Items { get; } + public IDictionary? Items { get; } } // This duck types with tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/GrpcBindingsFeatureStruct.cs From 40769d0f6ae029f4e2f7e1a3996d63a0b8bba318 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Wed, 22 Apr 2026 13:56:15 -0400 Subject: [PATCH 40/42] Update snapshots for aspnet_core.request on exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set Error: 1, error tags, and http.status_code 500 on the aspnet_core.request span when the isolated worker function throws. 🤖 Co-Authored-By: Claude Code --- ...zureFunctionsTests.Isolated.V4.AspNetCore.verified.txt | 8 +++++++- ...ureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt index a622ccc624fd..ec3f14a1c8fb 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore.verified.txt @@ -151,6 +151,7 @@ Service: AzureFunctionsAllTriggers, Type: web, ParentId: Id_11, + Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -164,10 +165,15 @@ aspnet_core.route: api/exception, component: aspnet_core, env: integration_tests, + error.msg: Task failed successfully., + error.stack: +System.InvalidOperationException: Task failed successfully. +at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), + error.type: System.InvalidOperationException, http.method: GET, http.request.headers.host: localhost:00000, http.route: api/exception, - http.status_code: 200, + http.status_code: 500, http.url: http://localhost:00000/api/exception, language: dotnet, runtime-id: Guid_1, diff --git a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt index e98ebf770d2d..4180c5900ea3 100644 --- a/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt +++ b/tracer/test/snapshots/AzureFunctionsTests.Isolated.V4.AspNetCore1.verified.txt @@ -151,6 +151,7 @@ Service: AzureFunctionsAllTriggers, Type: web, ParentId: Id_11, + Error: 1, Tags: { aas.environment.extension_version: unknown, aas.environment.instance_id: unknown, @@ -164,10 +165,15 @@ aspnet_core.route: api/exception, component: aspnet_core, env: integration_tests, + error.msg: Task failed successfully., + error.stack: +System.InvalidOperationException: Task failed successfully. +at Samples.AzureFunctions.AllTriggers.AllTriggers.Exception(HttpRequestData req), + error.type: System.InvalidOperationException, http.method: GET, http.request.headers.host: localhost:00000, http.route: api/exception, - http.status_code: 200, + http.status_code: 500, http.url: http://localhost:00000/api/exception, language: dotnet, runtime-id: Guid_1, From 3af124a45e5de38e43c7896f7ec82b92161cb263 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 23 Apr 2026 13:44:01 -0400 Subject: [PATCH 41/42] fallback to gRPC headers Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Azure/Functions/AzureFunctionsCommon.cs | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index d3ba44774f5b..94c3651236ee 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -273,22 +273,34 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even // Detect ASP.NET Core integration by checking for HttpContext in FunctionContext.Items. // In ASP.NET Core mode, HTTP requests are proxied directly (not via gRPC). // The headers in the gRPC message are STALE (contain host's root span context). - // The key "HttpRequestContext" is set by FunctionsHttpProxyingMiddleware in the worker + // The key "HttpRequestContext" is set by FunctionsHttpProxyingMiddleware in the worker. + // Only skip gRPC extraction when the ASP.NET Core active scope bridge is also present. + const string activeScopeBridgeKey = "ActiveScope"; var isAspNetCoreIntegration = functionContext.Items?.ContainsKey(HttpRequestContextKey) == true; + var hasAspNetCoreActiveScopeBridge = functionContext.Items?.ContainsKey(activeScopeBridgeKey) == true; - if (isAspNetCoreIntegration) + if (isAspNetCoreIntegration && hasAspNetCoreActiveScopeBridge) { - // Skip gRPC header extraction in HTTP proxying mode. - // The gRPC message headers contain STALE context from the host process - // (the host's root span), so extracting them would create an incorrect parent. - // Instead, we rely on the HttpContext.Items bridge to get the ASP.NET Core scope. - Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected (HTTP proxying mode)"); + // Skip gRPC header extraction in HTTP proxying mode only when the ASP.NET Core + // scope bridge is available. The gRPC message headers contain STALE context + // from the host process (the host's root span), so extracting them would create + // an incorrect parent. + Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected and active scope bridge found"); } else { - // Only extract from gRPC message when NOT using ASP.NET Core integration + // Fall back to gRPC message extraction when not using ASP.NET Core integration, + // or when proxying is detected but the ASP.NET Core scope bridge is unavailable. extractedContext = ExtractPropagatedContextFromHttp(functionContext, entry.Key as string).MergeBaggageInto(Baggage.Current); - Log.Debug("Extracted trace context from gRPC message (non-ASP.NET Core mode)"); + + if (isAspNetCoreIntegration && !hasAspNetCoreActiveScopeBridge) + { + Log.Debug("HTTP trigger detected ASP.NET Core integration, but no active scope bridge was found. Falling back to gRPC header extraction for trace correlation."); + } + else + { + Log.Debug("Extracted trace context from gRPC message (non-ASP.NET Core mode)"); + } } break; From 7b4da9ca19334d14cb92ed196980b6a8d7d4d9fc Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Thu, 23 Apr 2026 13:59:23 -0400 Subject: [PATCH 42/42] Gate gRPC fallback on aspNetCoreScope, not a fake Items key --- .../Azure/Functions/AzureFunctionsCommon.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs index 94c3651236ee..7b6ea4280fe0 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs @@ -274,18 +274,16 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even // In ASP.NET Core mode, HTTP requests are proxied directly (not via gRPC). // The headers in the gRPC message are STALE (contain host's root span context). // The key "HttpRequestContext" is set by FunctionsHttpProxyingMiddleware in the worker. - // Only skip gRPC extraction when the ASP.NET Core active scope bridge is also present. - const string activeScopeBridgeKey = "ActiveScope"; + // Only skip gRPC extraction when we successfully retrieved the ASP.NET Core scope bridge, + // otherwise fall back to the (stale) gRPC headers to at least keep the spans in the same trace. var isAspNetCoreIntegration = functionContext.Items?.ContainsKey(HttpRequestContextKey) == true; - var hasAspNetCoreActiveScopeBridge = functionContext.Items?.ContainsKey(activeScopeBridgeKey) == true; - if (isAspNetCoreIntegration && hasAspNetCoreActiveScopeBridge) + if (isAspNetCoreIntegration && aspNetCoreScope is not null) { - // Skip gRPC header extraction in HTTP proxying mode only when the ASP.NET Core - // scope bridge is available. The gRPC message headers contain STALE context - // from the host process (the host's root span), so extracting them would create - // an incorrect parent. - Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected and active scope bridge found"); + // Skip gRPC header extraction in HTTP proxying mode. We already have the + // ASP.NET Core scope from the HttpContext.Items bridge and will use it as + // the parent below. + Log.Debug("Skipping header extraction - HTTP trigger with ASP.NET Core integration detected (HTTP proxying mode)"); } else { @@ -293,9 +291,12 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even // or when proxying is detected but the ASP.NET Core scope bridge is unavailable. extractedContext = ExtractPropagatedContextFromHttp(functionContext, entry.Key as string).MergeBaggageInto(Baggage.Current); - if (isAspNetCoreIntegration && !hasAspNetCoreActiveScopeBridge) + if (isAspNetCoreIntegration) { - Log.Debug("HTTP trigger detected ASP.NET Core integration, but no active scope bridge was found. Falling back to gRPC header extraction for trace correlation."); + // ASP.NET Core integration detected but the scope bridge was not available + // (GetAspNetCoreScope returned null). Fall back to gRPC headers: the parent + // will be the host's root span (wrong, but keeps the spans in the same trace). + Log.Debug("HTTP trigger detected ASP.NET Core integration, but no active scope was found. Falling back to gRPC header extraction for trace correlation."); } else {