From 1d66070b5dffe6c1fb468f85e44c16d661444da5 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:11:48 +0200 Subject: [PATCH 1/9] explanation AI --- .../for-ai/ConsoleDeadlockIssue.md | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/development/for-ai/ConsoleDeadlockIssue.md diff --git a/docs/development/for-ai/ConsoleDeadlockIssue.md b/docs/development/for-ai/ConsoleDeadlockIssue.md new file mode 100644 index 000000000000..b16fdc6318d0 --- /dev/null +++ b/docs/development/for-ai/ConsoleDeadlockIssue.md @@ -0,0 +1,109 @@ +# Console App Deadlock with ConfigurationManager Config Builders (APMS-19239) + +## ROOT CAUSE CONFIRMED (from customer memory dump, 2026-04-22) + +The customer provided a memory dump of the hung process (`C:\Temp\APMS-19239\Dump\ConsoleApp1.exe_260421_142451.dmp`). +Analysis with `dotnet-dump analyze` identified the **actual** deadlock, which is different from the +`ConfigurationManager` lock theory below. + +**The real deadlock is a classic CLR type-initializer (`.cctor`) deadlock between the Managed Loader's +`Startup` class and the `AppDomain.AssemblyResolve` handler it registers.** + +### Thread A (main thread — `0x4a78`) + +The native profiler's module-initializer IL runs at the top of `Program.Main()` and calls +`Assembly.CreateInstance("Datadog.Trace.ClrProfiler.Managed.Loader.Startup")`, which triggers the +`Startup..cctor()`. Inside the `.cctor`: + +``` +Startup..cctor() + TryInvokeManagedMethod("Datadog.Trace.ClrProfiler.Instrumentation", "Initialize", ...) + Activator.CreateInstance(InstrumentationLoader) + Instrumentation..cctor() + DatadogLogging.GetLoggerFor(...) + DatadogLogging..cctor() + GlobalSettings.get_Instance() + GlobalSettings..cctor() + GlobalSettings.CreateFromDefaultSources() + GlobalConfigurationSource.get_CreationResult() + GlobalConfigurationSource..cctor() + CreateDefaultConfigurationSource(...) + ConfigurationManager.get_AppSettings() ← triggers configBuilder + KeyValueConfigBuilder.ProcessConfigurationSection + KeyValueConfigBuilder.EnsureGreedyInitialized + AzureAppConfigurationBuilder.GetAllValues + Task<...>.GetResultCore(true) ← sync-over-async + Task.InternalWait(...) + Task.SpinThenBlockingWait(...) + ManualResetEventSlim.Wait(...) ← BLOCKED +``` + +**State**: main thread holds the type-init lock for `Startup` (and every type in the chain above, +most importantly `Startup` itself, because that's what was just instantiated). It is blocking on a +`Task` spawned by `AzureAppConfigurationBuilder`. + +### Thread B (ThreadPool thread — `0x2bd4`) + +Running the async continuation of `AzureAppConfigurationBuilder.GetAllValuesAsync` while it enumerates +the Azure App Config response and lazily builds a `SecretClient` for each Key Vault reference: + +``` +ThreadPoolWorkQueue.Dispatch + ... Azure.Core pipeline (HttpPipeline, RetryPolicy, BearerTokenAuthenticationPolicy, ...) ... + AzureAppConfigurationBuilder+d__39.MoveNext + AzureAppConfigurationBuilder.GetKeyVaultValue + AzureAppConfigurationBuilder+<>c__DisplayClass41_0.b__0(Uri) + new DefaultAzureCredential() + DefaultAzureCredentialFactory.CreateFullDefaultCredentialChain() + DefaultAzureCredentialFactory.CreateVisualStudioCodeCredential() + new VisualStudioCodeCredential(...) + CredentialOptionsMapper.GetBrokerOptions(...) + DefaultAzureCredentialFactory.TryCreateDevelopmentBrokerOptions(...) + Type.GetType("Microsoft.Identity.Client.Broker.PublicClientApplicationBuilderExtensions, ...") + RuntimeTypeHandle.GetTypeByName (via P/Invoke) + AppDomain.OnAssemblyResolveEvent + Startup.AssemblyResolve_ManagedProfilerDependencies ← static method on Startup + [HelperMethodFrame] ← BLOCKED waiting for + Startup..cctor to finish +``` + +**State**: threadpool thread is blocked waiting for `Startup`'s class initializer to complete (the CLR +requires the type to be initialized before its static methods can execute). But `Startup..cctor` is +running on Thread A. + +### The Deadlock + +- Thread A holds `Startup`'s type-init lock and waits for a `Task`. +- Thread B is running that `Task`; it needs to invoke a static method on `Startup` to resolve an + assembly, which requires `Startup`'s type-init lock. +- Neither thread can make progress. **Classic `.cctor` × sync-over-async deadlock.** + +This is why every earlier mitigation failed: + +- `IsLoadingConfigurationManagerAppSettings` guard — wrong mechanism. The deadlock is not about + re-entrancy into CallTarget, it is purely a `.cctor` + `AssemblyResolve` interaction. +- `Lazy` in `IntegrationOptions` / `IntegrationMapper` — wrong type chain. The + `.cctor` chain that matters is `Startup → Instrumentation → DatadogLogging → GlobalSettings → + GlobalConfigurationSource → ConfigurationManager.AppSettings`, and the blocker is on `Startup` + itself because that's where the `AssemblyResolve` handler is a static member. +- Skipping `ConfigurationManager.AppSettings` during static init — was on the right track but + incomplete: even if we don't call it directly, any sync-over-async work inside the `Startup.cctor` + chain that ends up resolving an assembly will deadlock. + +### Fix Direction + +Two orthogonal root-cause fixes, either of which would break the deadlock. Doing both is safer. + +1. **Move `AssemblyResolve_ManagedProfilerDependencies` off of `Startup`.** Put the handler on a + separate type (e.g. `ManagedProfilerAssemblyResolver`) that has no `.cctor` dependency on any + long-running initialization. Register it from `Startup..cctor` via a delegate to that other type's + static method. Then Thread B can invoke the handler without needing `Startup` to be fully + initialized. + +2. **Stop reading `ConfigurationManager.AppSettings` from inside `Startup..cctor`'s transitive chain + on .NET Framework.** The customer's repro needs it gone from the `GlobalConfigurationSource` + static-init path; a deferred read (on first actual use, not during class init) is enough. + +Fix #1 is the more general protection — it defends against *any* sync-over-async inside the +`.cctor` chain, not just the `ConfigurationManager.AppSettings` case. Fix #2 is a pragmatic +narrowing: `ConfigurationManager.AppSettings` is the specific trigger for this customer. From de2747f98604e270336d32b3ffefa3b58ffd347b Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:24:07 +0200 Subject: [PATCH 2/9] Fix --- .../for-ai/ConsoleDeadlockIssue.md | 16 ++ ...ManagedProfilerAssemblyResolver.NetCore.cs | 152 ++++++++++++++++++ ...edProfilerAssemblyResolver.NetFramework.cs | 71 ++++++++ .../ManagedProfilerAssemblyResolver.cs | 38 +++++ .../Startup.NetCore.cs | 114 +------------ .../Startup.NetFramework.cs | 53 ------ .../Startup.cs | 17 +- 7 files changed, 293 insertions(+), 168 deletions(-) create mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs create mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs create mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs diff --git a/docs/development/for-ai/ConsoleDeadlockIssue.md b/docs/development/for-ai/ConsoleDeadlockIssue.md index b16fdc6318d0..b086b9bc21ae 100644 --- a/docs/development/for-ai/ConsoleDeadlockIssue.md +++ b/docs/development/for-ai/ConsoleDeadlockIssue.md @@ -107,3 +107,19 @@ Two orthogonal root-cause fixes, either of which would break the deadlock. Doing Fix #1 is the more general protection — it defends against *any* sync-over-async inside the `.cctor` chain, not just the `ConfigurationManager.AppSettings` case. Fix #2 is a pragmatic narrowing: `ConfigurationManager.AppSettings` is the specific trigger for this customer. + + +## FIX SHIPPED (2026-04-22) + +Moved `AssemblyResolve_ManagedProfilerDependencies` (and the .NET Core ALC `Resolving` handler) +off `Startup` onto a new `ManagedProfilerAssemblyResolver` class in +`tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/`. + +The new class has a trivial `.cctor`, so a ThreadPool thread invoking the handler no longer has +to wait on `Startup..cctor`. `Startup..cctor` seeds the resolver's state (`ManagedProfilerDirectory`, +and on .NET Core the assembly cache) before subscribing, which forces the resolver's class-init to +complete on the main thread before any ThreadPool work can be scheduled. + +Verified locally against the repro in `C:\Temp\APMS-19239\Dump\ConsoleApp1Repro\` with both +`Datadog.Trace.Bundle.3.41.0` (pre-fix: hangs) and the local dd-trace-6 build (post-fix: reaches +`Main()` normally). \ No newline at end of file diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs new file mode 100644 index 000000000000..b3e9b9ca435a --- /dev/null +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs @@ -0,0 +1,152 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETCOREAPP + +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Runtime.Loader; + +namespace Datadog.Trace.ClrProfiler.Managed.Loader +{ + internal static partial class ManagedProfilerAssemblyResolver + { + private static readonly AssemblyLoadContext DependencyLoadContext = new ManagedProfilerAssemblyLoadContext(); + + private static CachedAssembly[]? _assemblies; + + internal static void PopulateAssemblyCache(string directory) + { + if (!Directory.Exists(directory)) + { + return; + } + + var assemblies = new List(); + foreach (var file in Directory.EnumerateFiles(directory, "*.dll", SearchOption.TopDirectoryOnly)) + { + assemblies.Add(new CachedAssembly(file, null)); + } + + _assemblies = [..assemblies]; + StartupLogger.Debug("Total number of assemblies: {0}", _assemblies.Length); + } + + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + { + return ResolveAssembly(args.Name); + } + + internal static Assembly? OnAssemblyLoadContextResolving(AssemblyLoadContext context, AssemblyName assemblyName) + { + return ResolveAssembly(assemblyName.Name); + } + + internal static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. This seems to have been fixed in + // .NET Core in the 2.0 servicing branch, so we should not see this + // occur but guard against it anyway. If we do see it, exit early + // so we don't cause infinite recursion. + if (string.Equals(assemblyName.Name, "System.Private.CoreLib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + StartupLogger.Debug("Assembly Resolve event received for: {0}. Searching in: {1}", name, ManagedProfilerDirectory); + var path = Path.Combine(ManagedProfilerDirectory ?? string.Empty, $"{assemblyName.Name}.dll"); + + if (IsDatadogAssembly(path, out var cachedAssembly)) + { + // The file exists in the Home folder... + if (cachedAssembly is not null) + { + // The assembly is already loaded. + StartupLogger.Debug("Loading from cache. [Path: {0}]", path); + return cachedAssembly; + } + + // Only load the main profiler into the default AssemblyLoadContext. + // If the NuGet package provides Datadog.Trace or other libraries, loading them is handled in the following two ways: + // 1) If the AssemblyVersion is greater than or equal to the version used by Datadog.Trace, the assembly + // will load successfully and will not invoke this resolve event. + // 2) If the AssemblyVersion is lower than the version used by Datadog.Trace, the assembly will fail to load + // and invoke this resolve event. It must be loaded in a separate AssemblyLoadContext since the application will only + // load the originally referenced version. + StartupLogger.Debug("Calling DependencyLoadContext.LoadFromAssemblyPath(\"{0}\")", path); + var assembly = DependencyLoadContext.LoadFromAssemblyPath(path); // Load unresolved framework and third-party dependencies into a custom AssemblyLoadContext + SetDatadogAssembly(path, assembly); + return assembly; + } + + // The file doesn't exist in the Home folder. + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; + } + + private static bool IsDatadogAssembly(string path, out Assembly? cachedAssembly) + { + if (_assemblies is null) + { + cachedAssembly = null; + return false; + } + + for (var i = 0; i < _assemblies.Length; i++) + { + var assembly = _assemblies[i]; + if (assembly.Path == path) + { + cachedAssembly = assembly.Assembly; + return true; + } + } + + cachedAssembly = null; + return false; + } + + private static void SetDatadogAssembly(string path, Assembly cachedAssembly) + { + if (_assemblies is null) + { + return; + } + + for (var i = 0; i < _assemblies.Length; i++) + { + if (_assemblies[i].Path == path) + { + _assemblies[i] = new CachedAssembly(path, cachedAssembly); + return; + } + } + } + + private readonly struct CachedAssembly + { + public readonly string Path; + public readonly Assembly? Assembly; + + public CachedAssembly(string path, Assembly? assembly) + { + Path = path; + Assembly = assembly; + } + } + } +} + +#endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs new file mode 100644 index 000000000000..ccce09469d3f --- /dev/null +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs @@ -0,0 +1,71 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETFRAMEWORK + +#nullable enable + +using System; +using System.IO; +using System.Reflection; + +namespace Datadog.Trace.ClrProfiler.Managed.Loader +{ + internal static partial class ManagedProfilerAssemblyResolver + { + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + { + try + { + return ResolveAssembly(args.Name); + } + catch (Exception ex) + { + StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); + } + + return null; + } + + internal static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. Exit early so we don't cause + // infinite recursion. + if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); + StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); + + if (File.Exists(path)) + { + if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) + { + StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); + return null; + } + + StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); + var assembly = Assembly.LoadFrom(path); + StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); + return assembly; + } + + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; + } + } +} + +#endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs new file mode 100644 index 000000000000..d384bf16534a --- /dev/null +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs @@ -0,0 +1,38 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.ClrProfiler.Managed.Loader +{ + // This type owns the AppDomain.AssemblyResolve (and .NET Core + // AssemblyLoadContext.Resolving) callbacks that the tracer registers at + // startup. It is intentionally a separate static class from Startup so + // that invoking its static handlers never forces CLR type-initialization + // of Startup itself. + // + // Why that matters: on .NET Framework, if a configBuilder attached to + // (e.g. AzureAppConfigurationBuilder with useAzureKeyVault + // and DefaultAzureCredential) issues sync-over-async work during the + // Startup..cctor chain, the async continuation can run on a ThreadPool + // thread that needs to resolve a type (Type.GetType), which fires + // AppDomain.AssemblyResolve. If the handler lives on Startup, the + // ThreadPool thread has to wait for Startup..cctor to finish; the main + // thread is already blocked inside that .cctor waiting for the Task, + // which is waiting for the ThreadPool thread -> classic .cctor deadlock + // (APMS-19239). + // + // Keeping the handler on a class with no non-trivial .cctor means + // initialization of this type finishes on the main thread before the + // Task is scheduled, so any ThreadPool thread that later dispatches the + // handler sees the type as already initialized and runs without blocking. + internal static partial class ManagedProfilerAssemblyResolver + { + // Set by Startup..cctor before subscribing the handler below. + // An auto-property with no initializer keeps this type beforefieldinit + // and free of any meaningful class-init work. + internal static string? ManagedProfilerDirectory { get; set; } + } +} diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs index 3236809097e1..3c13a62be2a0 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs @@ -8,9 +8,7 @@ #nullable enable using System; -using System.Collections.Generic; using System.IO; -using System.Reflection; namespace Datadog.Trace.ClrProfiler.Managed.Loader { @@ -19,10 +17,6 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader /// public partial class Startup { - private static readonly System.Runtime.Loader.AssemblyLoadContext DependencyLoadContext = new ManagedProfilerAssemblyLoadContext(); - - private static CachedAssembly[]? _assemblies; - internal static string ComputeTfmDirectory(string tracerHomeDirectory) { var version = Environment.Version; @@ -46,114 +40,12 @@ internal static string ComputeTfmDirectory(string tracerHomeDirectory) var fullPath = Path.Combine(Path.GetFullPath(tracerHomeDirectory), managedLibrariesDirectory); - if (Directory.Exists(fullPath)) - { - // We use the List/Array approach due to the number of files in the tracer home folder (7 in netstandard, 2 netcoreapp3.1+) - var assemblies = new List(); - foreach (var file in Directory.EnumerateFiles(fullPath, "*.dll", SearchOption.TopDirectoryOnly)) - { - assemblies.Add(new CachedAssembly(file, null)); - } - - _assemblies = [..assemblies]; - StartupLogger.Debug("Total number of assemblies: {0}", _assemblies.Length); - } + // Populate the resolver's cache. The resolver is a separate type so its handler + // can be invoked from ThreadPool threads without having to wait on Startup..cctor. + ManagedProfilerAssemblyResolver.PopulateAssemblyCache(fullPath); return fullPath; } - - private static Assembly? AssemblyResolve_ManagedProfilerDependencies(object sender, ResolveEventArgs args) - { - return ResolveAssembly(args.Name); - } - - private static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); - - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. This seems to have been fixed in - // .NET Core in the 2.0 servicing branch, so we should not see this - // occur but guard against it anyway. If we do see it, exit early - // so we don't cause infinite recursion. - if (string.Equals(assemblyName.Name, "System.Private.CoreLib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - StartupLogger.Debug("Assembly Resolve event received for: {0}. Searching in: {1}", name, ManagedProfilerDirectory); - var path = Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); - - if (IsDatadogAssembly(path, out var cachedAssembly)) - { - // The file exists in the Home folder... - if (cachedAssembly is not null) - { - // The assembly is already loaded. - StartupLogger.Debug("Loading from cache. [Path: {0}]", path); - return cachedAssembly; - } - - // Only load the main profiler into the default AssemblyLoadContext. - // If the NuGet package provides Datadog.Trace or other libraries, loading them is handled in the following two ways: - // 1) If the AssemblyVersion is greater than or equal to the version used by Datadog.Trace, the assembly - // will load successfully and will not invoke this resolve event. - // 2) If the AssemblyVersion is lower than the version used by Datadog.Trace, the assembly will fail to load - // and invoke this resolve event. It must be loaded in a separate AssemblyLoadContext since the application will only - // load the originally referenced version. - StartupLogger.Debug("Calling DependencyLoadContext.LoadFromAssemblyPath(\"{0}\")", path); - var assembly = DependencyLoadContext.LoadFromAssemblyPath(path); // Load unresolved framework and third-party dependencies into a custom AssemblyLoadContext - SetDatadogAssembly(path, assembly); - return assembly; - } - - // The file doesn't exist in the Home folder. - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; - } - - private static bool IsDatadogAssembly(string path, out Assembly? cachedAssembly) - { - for (var i = 0; i < _assemblies!.Length; i++) - { - var assembly = _assemblies[i]; - if (assembly.Path == path) - { - cachedAssembly = assembly.Assembly; - return true; - } - } - - cachedAssembly = null; - return false; - } - - private static void SetDatadogAssembly(string path, Assembly cachedAssembly) - { - for (var i = 0; i < _assemblies!.Length; i++) - { - if (_assemblies[i].Path == path) - { - _assemblies[i] = new CachedAssembly(path, cachedAssembly); - return; - } - } - } - - private readonly struct CachedAssembly - { - public readonly string Path; - public readonly Assembly? Assembly; - - public CachedAssembly(string path, Assembly? assembly) - { - Path = path; - Assembly = assembly; - } - } } } diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs index 6668188c9084..5d669459eae9 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs @@ -7,9 +7,7 @@ #nullable enable -using System; using System.IO; -using System.Reflection; namespace Datadog.Trace.ClrProfiler.Managed.Loader { @@ -22,57 +20,6 @@ internal static string ComputeTfmDirectory(string tracerHomeDirectory) { return Path.Combine(Path.GetFullPath(tracerHomeDirectory), "net461"); } - - private static Assembly? AssemblyResolve_ManagedProfilerDependencies(object sender, ResolveEventArgs args) - { - try - { - return ResolveAssembly(args.Name); - } - catch (Exception ex) - { - StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); - } - - return null; - } - - private static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); - - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. Exit early so we don't cause - // infinite recursion. - if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); - StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); - - if (File.Exists(path)) - { - if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != AssemblyName) - { - StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, AssemblyName, path); - return null; - } - - StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); - var assembly = Assembly.LoadFrom(path); - StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); - return assembly; - } - - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; - } } } diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs index d203cb69f34a..d34cc994906c 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs @@ -17,7 +17,9 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader /// public sealed partial class Startup { - private const string AssemblyName = "Datadog.Trace, Version=3.43.0.0, Culture=neutral, PublicKeyToken=def86d061d0d2eeb"; + // internal so ManagedProfilerAssemblyResolver can reference it (const is inlined at compile time, + // so this does NOT create a runtime dependency on Startup's type initialization) + internal const string AssemblyName = "Datadog.Trace, Version=3.43.0.0, Culture=neutral, PublicKeyToken=def86d061d0d2eeb"; private const string AzureAppServicesSiteExtensionKey = "DD_AZURE_APP_SERVICES"; // only set when using the AAS site extension private const string TracerHomePathKey = "DD_DOTNET_TRACER_HOME"; @@ -76,9 +78,16 @@ static Startup() StartupLogger.Debug("Resolved Datadog.Trace.dll TFM directory to: {0}", ManagedProfilerDirectory); + // Publish the resolver's state BEFORE subscribing the handler. Reading/writing a + // static field of ManagedProfilerAssemblyResolver triggers its (trivial) type init + // here on the main thread, so by the time any ThreadPool thread later dispatches + // the handler, the type is already fully initialized and the dispatch doesn't have + // to wait on Startup..cctor. See APMS-19239. + ManagedProfilerAssemblyResolver.ManagedProfilerDirectory = ManagedProfilerDirectory; + try { - AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolve_ManagedProfilerDependencies; + AppDomain.CurrentDomain.AssemblyResolve += ManagedProfilerAssemblyResolver.OnAssemblyResolve; } catch (Exception ex) { @@ -88,7 +97,7 @@ static Startup() #if NETCOREAPP try { - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += (_, assemblyName) => ResolveAssembly(assemblyName.Name); + System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += ManagedProfilerAssemblyResolver.OnAssemblyLoadContextResolving; } catch (Exception ex) { @@ -181,7 +190,7 @@ private static void TryInvokeManagedMethod(string typeName, string methodName, s // We will try to resolve it manually as a last chance. StartupLogger.Log(ex, "Error on assembly load: {0}, Trying to solve it manually...", assemblyString); - var assembly = ResolveAssembly(assemblyString); + var assembly = ManagedProfilerAssemblyResolver.ResolveAssembly(assemblyString); if (assembly is not null) { StartupLogger.Log("Assembly '{0}' was resolved manually.", assemblyString); From 057fb8b056ccd7157de06c1e8355a10016808b97 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:34:07 +0200 Subject: [PATCH 3/9] minimal changes --- ...ManagedProfilerAssemblyResolver.NetCore.cs | 152 ------------------ ...edProfilerAssemblyResolver.NetFramework.cs | 26 ++- .../ManagedProfilerAssemblyResolver.cs | 38 ----- .../Startup.NetCore.cs | 114 ++++++++++++- .../Startup.cs | 23 ++- 5 files changed, 151 insertions(+), 202 deletions(-) delete mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs delete mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs deleted file mode 100644 index b3e9b9ca435a..000000000000 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetCore.cs +++ /dev/null @@ -1,152 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#if NETCOREAPP - -#nullable enable - -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Runtime.Loader; - -namespace Datadog.Trace.ClrProfiler.Managed.Loader -{ - internal static partial class ManagedProfilerAssemblyResolver - { - private static readonly AssemblyLoadContext DependencyLoadContext = new ManagedProfilerAssemblyLoadContext(); - - private static CachedAssembly[]? _assemblies; - - internal static void PopulateAssemblyCache(string directory) - { - if (!Directory.Exists(directory)) - { - return; - } - - var assemblies = new List(); - foreach (var file in Directory.EnumerateFiles(directory, "*.dll", SearchOption.TopDirectoryOnly)) - { - assemblies.Add(new CachedAssembly(file, null)); - } - - _assemblies = [..assemblies]; - StartupLogger.Debug("Total number of assemblies: {0}", _assemblies.Length); - } - - internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) - { - return ResolveAssembly(args.Name); - } - - internal static Assembly? OnAssemblyLoadContextResolving(AssemblyLoadContext context, AssemblyName assemblyName) - { - return ResolveAssembly(assemblyName.Name); - } - - internal static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); - - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. This seems to have been fixed in - // .NET Core in the 2.0 servicing branch, so we should not see this - // occur but guard against it anyway. If we do see it, exit early - // so we don't cause infinite recursion. - if (string.Equals(assemblyName.Name, "System.Private.CoreLib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - StartupLogger.Debug("Assembly Resolve event received for: {0}. Searching in: {1}", name, ManagedProfilerDirectory); - var path = Path.Combine(ManagedProfilerDirectory ?? string.Empty, $"{assemblyName.Name}.dll"); - - if (IsDatadogAssembly(path, out var cachedAssembly)) - { - // The file exists in the Home folder... - if (cachedAssembly is not null) - { - // The assembly is already loaded. - StartupLogger.Debug("Loading from cache. [Path: {0}]", path); - return cachedAssembly; - } - - // Only load the main profiler into the default AssemblyLoadContext. - // If the NuGet package provides Datadog.Trace or other libraries, loading them is handled in the following two ways: - // 1) If the AssemblyVersion is greater than or equal to the version used by Datadog.Trace, the assembly - // will load successfully and will not invoke this resolve event. - // 2) If the AssemblyVersion is lower than the version used by Datadog.Trace, the assembly will fail to load - // and invoke this resolve event. It must be loaded in a separate AssemblyLoadContext since the application will only - // load the originally referenced version. - StartupLogger.Debug("Calling DependencyLoadContext.LoadFromAssemblyPath(\"{0}\")", path); - var assembly = DependencyLoadContext.LoadFromAssemblyPath(path); // Load unresolved framework and third-party dependencies into a custom AssemblyLoadContext - SetDatadogAssembly(path, assembly); - return assembly; - } - - // The file doesn't exist in the Home folder. - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; - } - - private static bool IsDatadogAssembly(string path, out Assembly? cachedAssembly) - { - if (_assemblies is null) - { - cachedAssembly = null; - return false; - } - - for (var i = 0; i < _assemblies.Length; i++) - { - var assembly = _assemblies[i]; - if (assembly.Path == path) - { - cachedAssembly = assembly.Assembly; - return true; - } - } - - cachedAssembly = null; - return false; - } - - private static void SetDatadogAssembly(string path, Assembly cachedAssembly) - { - if (_assemblies is null) - { - return; - } - - for (var i = 0; i < _assemblies.Length; i++) - { - if (_assemblies[i].Path == path) - { - _assemblies[i] = new CachedAssembly(path, cachedAssembly); - return; - } - } - } - - private readonly struct CachedAssembly - { - public readonly string Path; - public readonly Assembly? Assembly; - - public CachedAssembly(string path, Assembly? assembly) - { - Path = path; - Assembly = assembly; - } - } - } -} - -#endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs index ccce09469d3f..e82905fcc69c 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs @@ -13,8 +13,32 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader { - internal static partial class ManagedProfilerAssemblyResolver + // This type owns the AppDomain.AssemblyResolve callback that the tracer + // registers at startup on .NET Framework. It is intentionally a separate + // static class from Startup so that invoking its static handler never + // forces CLR type-initialization of Startup itself. + // + // Why that matters: if a configBuilder attached to (e.g. + // AzureAppConfigurationBuilder with useAzureKeyVault and DefaultAzureCredential) + // issues sync-over-async work during the Startup..cctor chain, the async + // continuation can run on a ThreadPool thread that needs to resolve a type + // (Type.GetType), which fires AppDomain.AssemblyResolve. If the handler + // lives on Startup, that ThreadPool thread has to wait for Startup..cctor + // to finish; the main thread is already blocked inside that .cctor waiting + // for the Task, which is waiting for the ThreadPool thread -> classic + // .cctor deadlock (APMS-19239). + // + // Keeping the handler on a class with a trivial .cctor means Startup..cctor + // finishes the resolver's init before subscribing, so any ThreadPool thread + // that later dispatches the handler sees the type as already initialized + // and runs without blocking. + internal static class ManagedProfilerAssemblyResolver { + // Set by Startup..cctor before subscribing the handler below. + // An auto-property with no initializer keeps this type beforefieldinit + // and free of any meaningful class-init work. + internal static string? ManagedProfilerDirectory { get; set; } + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) { try diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs deleted file mode 100644 index d384bf16534a..000000000000 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.cs +++ /dev/null @@ -1,38 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#nullable enable - -namespace Datadog.Trace.ClrProfiler.Managed.Loader -{ - // This type owns the AppDomain.AssemblyResolve (and .NET Core - // AssemblyLoadContext.Resolving) callbacks that the tracer registers at - // startup. It is intentionally a separate static class from Startup so - // that invoking its static handlers never forces CLR type-initialization - // of Startup itself. - // - // Why that matters: on .NET Framework, if a configBuilder attached to - // (e.g. AzureAppConfigurationBuilder with useAzureKeyVault - // and DefaultAzureCredential) issues sync-over-async work during the - // Startup..cctor chain, the async continuation can run on a ThreadPool - // thread that needs to resolve a type (Type.GetType), which fires - // AppDomain.AssemblyResolve. If the handler lives on Startup, the - // ThreadPool thread has to wait for Startup..cctor to finish; the main - // thread is already blocked inside that .cctor waiting for the Task, - // which is waiting for the ThreadPool thread -> classic .cctor deadlock - // (APMS-19239). - // - // Keeping the handler on a class with no non-trivial .cctor means - // initialization of this type finishes on the main thread before the - // Task is scheduled, so any ThreadPool thread that later dispatches the - // handler sees the type as already initialized and runs without blocking. - internal static partial class ManagedProfilerAssemblyResolver - { - // Set by Startup..cctor before subscribing the handler below. - // An auto-property with no initializer keeps this type beforefieldinit - // and free of any meaningful class-init work. - internal static string? ManagedProfilerDirectory { get; set; } - } -} diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs index 3c13a62be2a0..3236809097e1 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetCore.cs @@ -8,7 +8,9 @@ #nullable enable using System; +using System.Collections.Generic; using System.IO; +using System.Reflection; namespace Datadog.Trace.ClrProfiler.Managed.Loader { @@ -17,6 +19,10 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader /// public partial class Startup { + private static readonly System.Runtime.Loader.AssemblyLoadContext DependencyLoadContext = new ManagedProfilerAssemblyLoadContext(); + + private static CachedAssembly[]? _assemblies; + internal static string ComputeTfmDirectory(string tracerHomeDirectory) { var version = Environment.Version; @@ -40,12 +46,114 @@ internal static string ComputeTfmDirectory(string tracerHomeDirectory) var fullPath = Path.Combine(Path.GetFullPath(tracerHomeDirectory), managedLibrariesDirectory); - // Populate the resolver's cache. The resolver is a separate type so its handler - // can be invoked from ThreadPool threads without having to wait on Startup..cctor. - ManagedProfilerAssemblyResolver.PopulateAssemblyCache(fullPath); + if (Directory.Exists(fullPath)) + { + // We use the List/Array approach due to the number of files in the tracer home folder (7 in netstandard, 2 netcoreapp3.1+) + var assemblies = new List(); + foreach (var file in Directory.EnumerateFiles(fullPath, "*.dll", SearchOption.TopDirectoryOnly)) + { + assemblies.Add(new CachedAssembly(file, null)); + } + + _assemblies = [..assemblies]; + StartupLogger.Debug("Total number of assemblies: {0}", _assemblies.Length); + } return fullPath; } + + private static Assembly? AssemblyResolve_ManagedProfilerDependencies(object sender, ResolveEventArgs args) + { + return ResolveAssembly(args.Name); + } + + private static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. This seems to have been fixed in + // .NET Core in the 2.0 servicing branch, so we should not see this + // occur but guard against it anyway. If we do see it, exit early + // so we don't cause infinite recursion. + if (string.Equals(assemblyName.Name, "System.Private.CoreLib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + StartupLogger.Debug("Assembly Resolve event received for: {0}. Searching in: {1}", name, ManagedProfilerDirectory); + var path = Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); + + if (IsDatadogAssembly(path, out var cachedAssembly)) + { + // The file exists in the Home folder... + if (cachedAssembly is not null) + { + // The assembly is already loaded. + StartupLogger.Debug("Loading from cache. [Path: {0}]", path); + return cachedAssembly; + } + + // Only load the main profiler into the default AssemblyLoadContext. + // If the NuGet package provides Datadog.Trace or other libraries, loading them is handled in the following two ways: + // 1) If the AssemblyVersion is greater than or equal to the version used by Datadog.Trace, the assembly + // will load successfully and will not invoke this resolve event. + // 2) If the AssemblyVersion is lower than the version used by Datadog.Trace, the assembly will fail to load + // and invoke this resolve event. It must be loaded in a separate AssemblyLoadContext since the application will only + // load the originally referenced version. + StartupLogger.Debug("Calling DependencyLoadContext.LoadFromAssemblyPath(\"{0}\")", path); + var assembly = DependencyLoadContext.LoadFromAssemblyPath(path); // Load unresolved framework and third-party dependencies into a custom AssemblyLoadContext + SetDatadogAssembly(path, assembly); + return assembly; + } + + // The file doesn't exist in the Home folder. + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; + } + + private static bool IsDatadogAssembly(string path, out Assembly? cachedAssembly) + { + for (var i = 0; i < _assemblies!.Length; i++) + { + var assembly = _assemblies[i]; + if (assembly.Path == path) + { + cachedAssembly = assembly.Assembly; + return true; + } + } + + cachedAssembly = null; + return false; + } + + private static void SetDatadogAssembly(string path, Assembly cachedAssembly) + { + for (var i = 0; i < _assemblies!.Length; i++) + { + if (_assemblies[i].Path == path) + { + _assemblies[i] = new CachedAssembly(path, cachedAssembly); + return; + } + } + } + + private readonly struct CachedAssembly + { + public readonly string Path; + public readonly Assembly? Assembly; + + public CachedAssembly(string path, Assembly? assembly) + { + Path = path; + Assembly = assembly; + } + } } } diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs index d34cc994906c..5c0b253b8b7a 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs @@ -78,16 +78,19 @@ static Startup() StartupLogger.Debug("Resolved Datadog.Trace.dll TFM directory to: {0}", ManagedProfilerDirectory); - // Publish the resolver's state BEFORE subscribing the handler. Reading/writing a - // static field of ManagedProfilerAssemblyResolver triggers its (trivial) type init - // here on the main thread, so by the time any ThreadPool thread later dispatches - // the handler, the type is already fully initialized and the dispatch doesn't have - // to wait on Startup..cctor. See APMS-19239. - ManagedProfilerAssemblyResolver.ManagedProfilerDirectory = ManagedProfilerDirectory; - try { +#if NETFRAMEWORK + // On .NET Framework, route AssemblyResolve through a class other than Startup so + // the handler doesn't require Startup..cctor to have finished. If a configBuilder + // on issues sync-over-async work during Startup..cctor, the async + // continuation may fire AssemblyResolve on a ThreadPool thread; a handler on + // Startup itself would deadlock waiting on Startup..cctor. See APMS-19239. + ManagedProfilerAssemblyResolver.ManagedProfilerDirectory = ManagedProfilerDirectory; AppDomain.CurrentDomain.AssemblyResolve += ManagedProfilerAssemblyResolver.OnAssemblyResolve; +#else + AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolve_ManagedProfilerDependencies; +#endif } catch (Exception ex) { @@ -97,7 +100,7 @@ static Startup() #if NETCOREAPP try { - System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += ManagedProfilerAssemblyResolver.OnAssemblyLoadContextResolving; + System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += (_, assemblyName) => ResolveAssembly(assemblyName.Name); } catch (Exception ex) { @@ -190,7 +193,11 @@ private static void TryInvokeManagedMethod(string typeName, string methodName, s // We will try to resolve it manually as a last chance. StartupLogger.Log(ex, "Error on assembly load: {0}, Trying to solve it manually...", assemblyString); +#if NETFRAMEWORK var assembly = ManagedProfilerAssemblyResolver.ResolveAssembly(assemblyString); +#else + var assembly = ResolveAssembly(assemblyString); +#endif if (assembly is not null) { StartupLogger.Log("Assembly '{0}' was resolved manually.", assemblyString); From 90dbae6d8e41d6b86bb4a36fb2dfe3b52478cde0 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:38:36 +0200 Subject: [PATCH 4/9] Minimal. --- ...edProfilerAssemblyResolver.NetFramework.cs | 95 ------------------- .../Startup.NetFramework.cs | 67 +++++++++++++ .../Startup.cs | 2 +- 3 files changed, 68 insertions(+), 96 deletions(-) delete mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs deleted file mode 100644 index e82905fcc69c..000000000000 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs +++ /dev/null @@ -1,95 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// - -#if NETFRAMEWORK - -#nullable enable - -using System; -using System.IO; -using System.Reflection; - -namespace Datadog.Trace.ClrProfiler.Managed.Loader -{ - // This type owns the AppDomain.AssemblyResolve callback that the tracer - // registers at startup on .NET Framework. It is intentionally a separate - // static class from Startup so that invoking its static handler never - // forces CLR type-initialization of Startup itself. - // - // Why that matters: if a configBuilder attached to (e.g. - // AzureAppConfigurationBuilder with useAzureKeyVault and DefaultAzureCredential) - // issues sync-over-async work during the Startup..cctor chain, the async - // continuation can run on a ThreadPool thread that needs to resolve a type - // (Type.GetType), which fires AppDomain.AssemblyResolve. If the handler - // lives on Startup, that ThreadPool thread has to wait for Startup..cctor - // to finish; the main thread is already blocked inside that .cctor waiting - // for the Task, which is waiting for the ThreadPool thread -> classic - // .cctor deadlock (APMS-19239). - // - // Keeping the handler on a class with a trivial .cctor means Startup..cctor - // finishes the resolver's init before subscribing, so any ThreadPool thread - // that later dispatches the handler sees the type as already initialized - // and runs without blocking. - internal static class ManagedProfilerAssemblyResolver - { - // Set by Startup..cctor before subscribing the handler below. - // An auto-property with no initializer keeps this type beforefieldinit - // and free of any meaningful class-init work. - internal static string? ManagedProfilerDirectory { get; set; } - - internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) - { - try - { - return ResolveAssembly(args.Name); - } - catch (Exception ex) - { - StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); - } - - return null; - } - - internal static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); - - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. Exit early so we don't cause - // infinite recursion. - if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); - StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); - - if (File.Exists(path)) - { - if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) - { - StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); - return null; - } - - StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); - var assembly = Assembly.LoadFrom(path); - StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); - return assembly; - } - - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; - } - } -} - -#endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs index 5d669459eae9..b0d0f1032735 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs @@ -7,7 +7,9 @@ #nullable enable +using System; using System.IO; +using System.Reflection; namespace Datadog.Trace.ClrProfiler.Managed.Loader { @@ -21,6 +23,71 @@ internal static string ComputeTfmDirectory(string tracerHomeDirectory) return Path.Combine(Path.GetFullPath(tracerHomeDirectory), "net461"); } } + + // Owns the AppDomain.AssemblyResolve callback on .NET Framework. Kept on a + // separate class from Startup so the handler can be dispatched without + // forcing Startup's type-initializer to have finished. If the handler were + // a method on Startup, a configBuilder attached to that + // issues sync-over-async work during Startup..cctor could deadlock: the + // main thread would be blocked inside the .cctor waiting for a Task, whose + // continuation runs on a ThreadPool thread that probes Type.GetType and + // fires AssemblyResolve, whose handler would then wait on Startup..cctor. + internal static class ManagedProfilerAssemblyResolver + { + // Seeded by Startup..cctor before the handler is subscribed. + internal static string? ManagedProfilerDirectory { get; set; } + + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + { + try + { + return ResolveAssembly(args.Name); + } + catch (Exception ex) + { + StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); + } + + return null; + } + + internal static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. Exit early so we don't cause + // infinite recursion. + if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); + StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); + + if (File.Exists(path)) + { + if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) + { + StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); + return null; + } + + StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); + var assembly = Assembly.LoadFrom(path); + StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); + return assembly; + } + + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; + } + } } #endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs index 5c0b253b8b7a..9940618c3b70 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs @@ -85,7 +85,7 @@ static Startup() // the handler doesn't require Startup..cctor to have finished. If a configBuilder // on issues sync-over-async work during Startup..cctor, the async // continuation may fire AssemblyResolve on a ThreadPool thread; a handler on - // Startup itself would deadlock waiting on Startup..cctor. See APMS-19239. + // Startup itself would deadlock waiting on Startup..cctor. ManagedProfilerAssemblyResolver.ManagedProfilerDirectory = ManagedProfilerDirectory; AppDomain.CurrentDomain.AssemblyResolve += ManagedProfilerAssemblyResolver.OnAssemblyResolve; #else From 28693cc33af3acb215226ee4ce89183f96f8a500 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:50:27 +0200 Subject: [PATCH 5/9] Fix compilation error --- ...edProfilerAssemblyResolver.NetFramework.cs | 82 +++++++++++++++++++ .../Startup.NetFramework.cs | 67 --------------- 2 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs new file mode 100644 index 000000000000..c69b530177ee --- /dev/null +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs @@ -0,0 +1,82 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NETFRAMEWORK + +#nullable enable + +using System; +using System.IO; +using System.Reflection; + +namespace Datadog.Trace.ClrProfiler.Managed.Loader +{ + // Owns the AppDomain.AssemblyResolve callback on .NET Framework. Kept on a + // separate class from Startup so the handler can be dispatched without + // forcing Startup's type-initializer to have finished. If the handler were + // a method on Startup, a configBuilder attached to that + // issues sync-over-async work during Startup..cctor could deadlock: the + // main thread would be blocked inside the .cctor waiting for a Task, whose + // continuation runs on a ThreadPool thread that probes Type.GetType and + // fires AssemblyResolve, whose handler would then wait on Startup..cctor. + internal static class ManagedProfilerAssemblyResolver + { + // Seeded by Startup..cctor before the handler is subscribed. + internal static string? ManagedProfilerDirectory { get; set; } + + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + { + try + { + return ResolveAssembly(args.Name); + } + catch (Exception ex) + { + StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); + } + + return null; + } + + internal static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. Exit early so we don't cause + // infinite recursion. + if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); + StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); + + if (File.Exists(path)) + { + if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) + { + StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); + return null; + } + + StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); + var assembly = Assembly.LoadFrom(path); + StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); + return assembly; + } + + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; + } + } +} + +#endif diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs index b0d0f1032735..5d669459eae9 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.NetFramework.cs @@ -7,9 +7,7 @@ #nullable enable -using System; using System.IO; -using System.Reflection; namespace Datadog.Trace.ClrProfiler.Managed.Loader { @@ -23,71 +21,6 @@ internal static string ComputeTfmDirectory(string tracerHomeDirectory) return Path.Combine(Path.GetFullPath(tracerHomeDirectory), "net461"); } } - - // Owns the AppDomain.AssemblyResolve callback on .NET Framework. Kept on a - // separate class from Startup so the handler can be dispatched without - // forcing Startup's type-initializer to have finished. If the handler were - // a method on Startup, a configBuilder attached to that - // issues sync-over-async work during Startup..cctor could deadlock: the - // main thread would be blocked inside the .cctor waiting for a Task, whose - // continuation runs on a ThreadPool thread that probes Type.GetType and - // fires AssemblyResolve, whose handler would then wait on Startup..cctor. - internal static class ManagedProfilerAssemblyResolver - { - // Seeded by Startup..cctor before the handler is subscribed. - internal static string? ManagedProfilerDirectory { get; set; } - - internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) - { - try - { - return ResolveAssembly(args.Name); - } - catch (Exception ex) - { - StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); - } - - return null; - } - - internal static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); - - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. Exit early so we don't cause - // infinite recursion. - if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); - StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); - - if (File.Exists(path)) - { - if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) - { - StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); - return null; - } - - StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); - var assembly = Assembly.LoadFrom(path); - StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); - return assembly; - } - - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; - } - } } #endif From 50f1457ce27c3efc6248e1dab91ed57562766359 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 14:51:58 +0200 Subject: [PATCH 6/9] scoped namespace --- ...edProfilerAssemblyResolver.NetFramework.cs | 103 +++++++++--------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs index c69b530177ee..5d8639bc47ca 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/ManagedProfilerAssemblyResolver.NetFramework.cs @@ -11,71 +11,70 @@ using System.IO; using System.Reflection; -namespace Datadog.Trace.ClrProfiler.Managed.Loader +namespace Datadog.Trace.ClrProfiler.Managed.Loader; + +// Owns the AppDomain.AssemblyResolve callback on .NET Framework. Kept on a +// separate class from Startup so the handler can be dispatched without +// forcing Startup's type-initializer to have finished. If the handler were +// a method on Startup, a configBuilder attached to that +// issues sync-over-async work during Startup..cctor could deadlock: the +// main thread would be blocked inside the .cctor waiting for a Task, whose +// continuation runs on a ThreadPool thread that probes Type.GetType and +// fires AssemblyResolve, whose handler would then wait on Startup..cctor. +internal static class ManagedProfilerAssemblyResolver { - // Owns the AppDomain.AssemblyResolve callback on .NET Framework. Kept on a - // separate class from Startup so the handler can be dispatched without - // forcing Startup's type-initializer to have finished. If the handler were - // a method on Startup, a configBuilder attached to that - // issues sync-over-async work during Startup..cctor could deadlock: the - // main thread would be blocked inside the .cctor waiting for a Task, whose - // continuation runs on a ThreadPool thread that probes Type.GetType and - // fires AssemblyResolve, whose handler would then wait on Startup..cctor. - internal static class ManagedProfilerAssemblyResolver - { - // Seeded by Startup..cctor before the handler is subscribed. - internal static string? ManagedProfilerDirectory { get; set; } + // Seeded by Startup..cctor before the handler is subscribed. + internal static string? ManagedProfilerDirectory { get; set; } - internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + internal static Assembly? OnAssemblyResolve(object sender, ResolveEventArgs args) + { + try { - try - { - return ResolveAssembly(args.Name); - } - catch (Exception ex) - { - StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); - } + return ResolveAssembly(args.Name); + } + catch (Exception ex) + { + StartupLogger.Log(ex, "Error resolving assembly: {0}", args.Name); + } + + return null; + } + internal static Assembly? ResolveAssembly(string name) + { + var assemblyName = new AssemblyName(name); + + // On .NET Framework, having a non-US locale can cause mscorlib + // to enter the AssemblyResolve event when searching for resources + // in its satellite assemblies. Exit early so we don't cause + // infinite recursion. + if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || + string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) + { return null; } - internal static Assembly? ResolveAssembly(string name) - { - var assemblyName = new AssemblyName(name); + // WARNING: Logs must not be added _before_ we check for the above bail-out conditions + var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); + StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); - // On .NET Framework, having a non-US locale can cause mscorlib - // to enter the AssemblyResolve event when searching for resources - // in its satellite assemblies. Exit early so we don't cause - // infinite recursion. - if (string.Equals(assemblyName.Name, "mscorlib.resources", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "System.Net.Http", StringComparison.OrdinalIgnoreCase) || - string.Equals(assemblyName.Name, "vstest.console.resources", StringComparison.OrdinalIgnoreCase)) + if (File.Exists(path)) + { + if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) { + StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); return null; } - // WARNING: Logs must not be added _before_ we check for the above bail-out conditions - var path = string.IsNullOrEmpty(ManagedProfilerDirectory) ? $"{assemblyName.Name}.dll" : Path.Combine(ManagedProfilerDirectory, $"{assemblyName.Name}.dll"); - StartupLogger.Debug("Assembly Resolve event received for: {0}. Looking for: {1}", name, path); - - if (File.Exists(path)) - { - if (name.StartsWith("Datadog.Trace, Version=", StringComparison.Ordinal) && name != Startup.AssemblyName) - { - StartupLogger.Debug(" Trying to load '{0}' which does not match the expected version ('{1}'). [Path={2}]", name, Startup.AssemblyName, path); - return null; - } - - StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); - var assembly = Assembly.LoadFrom(path); - StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); - return assembly; - } - - StartupLogger.Debug("Assembly not found in path: {0}", path); - return null; + StartupLogger.Debug("Calling Assembly.LoadFrom(\"{0}\")", path); + var assembly = Assembly.LoadFrom(path); + StartupLogger.Debug("Assembly loaded: {0}", assembly.FullName); + return assembly; } + + StartupLogger.Debug("Assembly not found in path: {0}", path); + return null; } } From 4cca3d08598ab3048b4c4b3a98441adb2931c777 Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 15:06:04 +0200 Subject: [PATCH 7/9] Drop ConsoleDeadlockIssue.md investigation notes Investigation-only notes with workstation-specific paths; PR description carries the relevant context. --- .../for-ai/ConsoleDeadlockIssue.md | 125 ------------------ 1 file changed, 125 deletions(-) delete mode 100644 docs/development/for-ai/ConsoleDeadlockIssue.md diff --git a/docs/development/for-ai/ConsoleDeadlockIssue.md b/docs/development/for-ai/ConsoleDeadlockIssue.md deleted file mode 100644 index b086b9bc21ae..000000000000 --- a/docs/development/for-ai/ConsoleDeadlockIssue.md +++ /dev/null @@ -1,125 +0,0 @@ -# Console App Deadlock with ConfigurationManager Config Builders (APMS-19239) - -## ROOT CAUSE CONFIRMED (from customer memory dump, 2026-04-22) - -The customer provided a memory dump of the hung process (`C:\Temp\APMS-19239\Dump\ConsoleApp1.exe_260421_142451.dmp`). -Analysis with `dotnet-dump analyze` identified the **actual** deadlock, which is different from the -`ConfigurationManager` lock theory below. - -**The real deadlock is a classic CLR type-initializer (`.cctor`) deadlock between the Managed Loader's -`Startup` class and the `AppDomain.AssemblyResolve` handler it registers.** - -### Thread A (main thread — `0x4a78`) - -The native profiler's module-initializer IL runs at the top of `Program.Main()` and calls -`Assembly.CreateInstance("Datadog.Trace.ClrProfiler.Managed.Loader.Startup")`, which triggers the -`Startup..cctor()`. Inside the `.cctor`: - -``` -Startup..cctor() - TryInvokeManagedMethod("Datadog.Trace.ClrProfiler.Instrumentation", "Initialize", ...) - Activator.CreateInstance(InstrumentationLoader) - Instrumentation..cctor() - DatadogLogging.GetLoggerFor(...) - DatadogLogging..cctor() - GlobalSettings.get_Instance() - GlobalSettings..cctor() - GlobalSettings.CreateFromDefaultSources() - GlobalConfigurationSource.get_CreationResult() - GlobalConfigurationSource..cctor() - CreateDefaultConfigurationSource(...) - ConfigurationManager.get_AppSettings() ← triggers configBuilder - KeyValueConfigBuilder.ProcessConfigurationSection - KeyValueConfigBuilder.EnsureGreedyInitialized - AzureAppConfigurationBuilder.GetAllValues - Task<...>.GetResultCore(true) ← sync-over-async - Task.InternalWait(...) - Task.SpinThenBlockingWait(...) - ManualResetEventSlim.Wait(...) ← BLOCKED -``` - -**State**: main thread holds the type-init lock for `Startup` (and every type in the chain above, -most importantly `Startup` itself, because that's what was just instantiated). It is blocking on a -`Task` spawned by `AzureAppConfigurationBuilder`. - -### Thread B (ThreadPool thread — `0x2bd4`) - -Running the async continuation of `AzureAppConfigurationBuilder.GetAllValuesAsync` while it enumerates -the Azure App Config response and lazily builds a `SecretClient` for each Key Vault reference: - -``` -ThreadPoolWorkQueue.Dispatch - ... Azure.Core pipeline (HttpPipeline, RetryPolicy, BearerTokenAuthenticationPolicy, ...) ... - AzureAppConfigurationBuilder+d__39.MoveNext - AzureAppConfigurationBuilder.GetKeyVaultValue - AzureAppConfigurationBuilder+<>c__DisplayClass41_0.b__0(Uri) - new DefaultAzureCredential() - DefaultAzureCredentialFactory.CreateFullDefaultCredentialChain() - DefaultAzureCredentialFactory.CreateVisualStudioCodeCredential() - new VisualStudioCodeCredential(...) - CredentialOptionsMapper.GetBrokerOptions(...) - DefaultAzureCredentialFactory.TryCreateDevelopmentBrokerOptions(...) - Type.GetType("Microsoft.Identity.Client.Broker.PublicClientApplicationBuilderExtensions, ...") - RuntimeTypeHandle.GetTypeByName (via P/Invoke) - AppDomain.OnAssemblyResolveEvent - Startup.AssemblyResolve_ManagedProfilerDependencies ← static method on Startup - [HelperMethodFrame] ← BLOCKED waiting for - Startup..cctor to finish -``` - -**State**: threadpool thread is blocked waiting for `Startup`'s class initializer to complete (the CLR -requires the type to be initialized before its static methods can execute). But `Startup..cctor` is -running on Thread A. - -### The Deadlock - -- Thread A holds `Startup`'s type-init lock and waits for a `Task`. -- Thread B is running that `Task`; it needs to invoke a static method on `Startup` to resolve an - assembly, which requires `Startup`'s type-init lock. -- Neither thread can make progress. **Classic `.cctor` × sync-over-async deadlock.** - -This is why every earlier mitigation failed: - -- `IsLoadingConfigurationManagerAppSettings` guard — wrong mechanism. The deadlock is not about - re-entrancy into CallTarget, it is purely a `.cctor` + `AssemblyResolve` interaction. -- `Lazy` in `IntegrationOptions` / `IntegrationMapper` — wrong type chain. The - `.cctor` chain that matters is `Startup → Instrumentation → DatadogLogging → GlobalSettings → - GlobalConfigurationSource → ConfigurationManager.AppSettings`, and the blocker is on `Startup` - itself because that's where the `AssemblyResolve` handler is a static member. -- Skipping `ConfigurationManager.AppSettings` during static init — was on the right track but - incomplete: even if we don't call it directly, any sync-over-async work inside the `Startup.cctor` - chain that ends up resolving an assembly will deadlock. - -### Fix Direction - -Two orthogonal root-cause fixes, either of which would break the deadlock. Doing both is safer. - -1. **Move `AssemblyResolve_ManagedProfilerDependencies` off of `Startup`.** Put the handler on a - separate type (e.g. `ManagedProfilerAssemblyResolver`) that has no `.cctor` dependency on any - long-running initialization. Register it from `Startup..cctor` via a delegate to that other type's - static method. Then Thread B can invoke the handler without needing `Startup` to be fully - initialized. - -2. **Stop reading `ConfigurationManager.AppSettings` from inside `Startup..cctor`'s transitive chain - on .NET Framework.** The customer's repro needs it gone from the `GlobalConfigurationSource` - static-init path; a deferred read (on first actual use, not during class init) is enough. - -Fix #1 is the more general protection — it defends against *any* sync-over-async inside the -`.cctor` chain, not just the `ConfigurationManager.AppSettings` case. Fix #2 is a pragmatic -narrowing: `ConfigurationManager.AppSettings` is the specific trigger for this customer. - - -## FIX SHIPPED (2026-04-22) - -Moved `AssemblyResolve_ManagedProfilerDependencies` (and the .NET Core ALC `Resolving` handler) -off `Startup` onto a new `ManagedProfilerAssemblyResolver` class in -`tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/`. - -The new class has a trivial `.cctor`, so a ThreadPool thread invoking the handler no longer has -to wait on `Startup..cctor`. `Startup..cctor` seeds the resolver's state (`ManagedProfilerDirectory`, -and on .NET Core the assembly cache) before subscribing, which forces the resolver's class-init to -complete on the main thread before any ThreadPool work can be scheduled. - -Verified locally against the repro in `C:\Temp\APMS-19239\Dump\ConsoleApp1Repro\` with both -`Datadog.Trace.Bundle.3.41.0` (pre-fix: hangs) and the local dd-trace-6 build (post-fix: reaches -`Main()` normally). \ No newline at end of file From dd7351aa8131953292ba2e36047386ba48c3878d Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 15:51:04 +0200 Subject: [PATCH 8/9] Add warning comment on AssemblyName re: cctor deadlock --- .../Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs index 9940618c3b70..106a32699450 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs @@ -17,8 +17,13 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader /// public sealed partial class Startup { - // internal so ManagedProfilerAssemblyResolver can reference it (const is inlined at compile time, - // so this does NOT create a runtime dependency on Startup's type initialization) + // internal so ManagedProfilerAssemblyResolver can reference it. Safe because const strings are + // inlined at compile time, so this does NOT create a runtime dependency on Startup's type + // initializer. + // WARNING: do NOT add non-const static members on Startup that ManagedProfilerAssemblyResolver + // references. A non-const access would force Startup..cctor to complete before the + // AssemblyResolve handler could run, re-introducing the .cctor x sync-over-async deadlock + // that this split was designed to prevent. internal const string AssemblyName = "Datadog.Trace, Version=3.43.0.0, Culture=neutral, PublicKeyToken=def86d061d0d2eeb"; private const string AzureAppServicesSiteExtensionKey = "DD_AZURE_APP_SERVICES"; // only set when using the AAS site extension private const string TracerHomePathKey = "DD_DOTNET_TRACER_HOME"; From 862f3a0a77fef03b14f42f247c08e5d26c06d74e Mon Sep 17 00:00:00 2001 From: NachoEchevarria Date: Wed, 22 Apr 2026 16:06:25 +0200 Subject: [PATCH 9/9] Decrease comment size --- .../Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs index 106a32699450..9f7a8a6a8336 100644 --- a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs +++ b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs @@ -17,13 +17,8 @@ namespace Datadog.Trace.ClrProfiler.Managed.Loader /// public sealed partial class Startup { - // internal so ManagedProfilerAssemblyResolver can reference it. Safe because const strings are - // inlined at compile time, so this does NOT create a runtime dependency on Startup's type - // initializer. - // WARNING: do NOT add non-const static members on Startup that ManagedProfilerAssemblyResolver - // references. A non-const access would force Startup..cctor to complete before the - // AssemblyResolve handler could run, re-introducing the .cctor x sync-over-async deadlock - // that this split was designed to prevent. + // internal so ManagedProfilerAssemblyResolver can reference it. Safe because const strings are inlined at compile time. + // Do not add non-const static members on Startup that the resolver needs - that would re-introduce the .cctor deadlock. internal const string AssemblyName = "Datadog.Trace, Version=3.43.0.0, Culture=neutral, PublicKeyToken=def86d061d0d2eeb"; private const string AzureAppServicesSiteExtensionKey = "DD_AZURE_APP_SERVICES"; // only set when using the AAS site extension private const string TracerHomePathKey = "DD_DOTNET_TRACER_HOME";