diff --git a/investigations/issue-3148/Issue3148PathProbe.csproj b/investigations/issue-3148/Issue3148PathProbe.csproj new file mode 100644 index 0000000000..748cd1faea --- /dev/null +++ b/investigations/issue-3148/Issue3148PathProbe.csproj @@ -0,0 +1,16 @@ + + + Exe + net8.0 + enable + enable + latest + false + true + true + + + + + + \ No newline at end of file diff --git a/investigations/issue-3148/Program.cs b/investigations/issue-3148/Program.cs new file mode 100644 index 0000000000..a12f3f15f0 --- /dev/null +++ b/investigations/issue-3148/Program.cs @@ -0,0 +1,1236 @@ +using System.Diagnostics; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; +using Microsoft.Data.SqlClient; + +internal static class Program +{ + private const string ChildModeEnvironmentVariable = "ISSUE3148_PATH_PROBE_CHILD"; + + private static int Main(string[] args) + { + Options options; + + try + { + options = Options.Parse(args); + } + catch (ArgumentException ex) + { + Console.Error.WriteLine(ex.Message); + PrintUsage(); + return 2; + } + + if (options.StressLoadAbsent) + { + return RunStressTest(options); + } + + if (options.ForceX86DotnetFirst) + { + return RunWithForcedDotnetPathOrdering(options); + } + + if (options.ProbeSniPaths) + { + return RunProbeSniPaths(); + } + + if (options.ProbeArchMatch) + { + return RunProbeArchMatch(); + } + + if (options.ProbeNativeLoad) + { + return RunProbeNativeLoad(); + } + + if (options.ProbeExtractionRace) + { + return RunProbeExtractionRace(options); + } + + if (options.ProbeLazyLoad) + { + return RunProbeLazyLoad(options); + } + + PrintEnvironmentReport(); + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + Console.WriteLine(); + Console.WriteLine("No connection string provided. Skipping SqlClient open test."); + return 0; + } + + Console.WriteLine(); + Console.WriteLine("Attempting SqlConnection.Open()..."); + + try + { + using SqlConnection connection = new(options.ConnectionString); + connection.Open(); + Console.WriteLine("SqlConnection.Open() succeeded."); + return 0; + } + catch (Exception ex) + { + Console.WriteLine("SqlConnection.Open() failed."); + PrintExceptionTree(ex); + return 1; + } + } + + private static void PrintEnvironmentReport() + { + Console.WriteLine("Issue 3148 PATH probe"); + Console.WriteLine(new string('=', 21)); + Console.WriteLine($"OS description: {RuntimeInformation.OSDescription}"); + Console.WriteLine($"OS architecture: {RuntimeInformation.OSArchitecture}"); + Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}"); + Console.WriteLine($"Framework: {RuntimeInformation.FrameworkDescription}"); + Console.WriteLine($"Current process path: {Environment.ProcessPath ?? ""}"); + Console.WriteLine($"Current directory: {Environment.CurrentDirectory}"); + Console.WriteLine(); + + string? pathValue = Environment.GetEnvironmentVariable("PATH"); + string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":"; + string[] pathEntries = (pathValue ?? string.Empty) + .Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + Console.WriteLine("PATH entries containing 'dotnet':"); + int dotnetEntryCount = 0; + + for (int index = 0; index < pathEntries.Length; index++) + { + if (pathEntries[index].Contains("dotnet", StringComparison.OrdinalIgnoreCase)) + { + dotnetEntryCount++; + Console.WriteLine($" [{index}] {pathEntries[index]}"); + } + } + + if (dotnetEntryCount == 0) + { + Console.WriteLine(" "); + } + + Console.WriteLine(); + Console.WriteLine($"Resolved dotnet executable from PATH: {ResolveCommandFromPath(GetDotnetCommandName()) ?? ""}"); + Console.WriteLine($"PATH ordering assessment: {AssessDotnetPathOrdering(pathEntries)}"); + } + + private static string GetDotnetCommandName() + => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet"; + + private static string? ResolveCommandFromPath(string commandName) + { + string? pathValue = Environment.GetEnvironmentVariable("PATH"); + return ResolveCommandFromPath(commandName, pathValue); + } + + private static string? ResolveCommandFromPath(string commandName, string? pathValue) + { + if (string.IsNullOrWhiteSpace(pathValue)) + { + return null; + } + + string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":"; + string[] pathEntries = pathValue.Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (string entry in pathEntries) + { + try + { + string candidate = Path.Combine(entry, commandName); + if (File.Exists(candidate)) + { + return candidate; + } + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + } + } + + return null; + } + + private static void PrintExceptionTree(Exception ex) + { + int depth = 0; + Exception? current = ex; + + while (current is not null) + { + Console.WriteLine($"[{depth}] {current.GetType().FullName}: {current.Message}"); + Console.WriteLine(current.StackTrace); + Console.WriteLine(); + current = current.InnerException; + depth++; + } + } + + private static string AssessDotnetPathOrdering(string[] pathEntries) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return "non-Windows environment; x86/x64 Windows PATH ordering check not applicable"; + } + + int x64Index = FindPathEntryIndex(pathEntries, "program files\\dotnet"); + int x86Index = FindPathEntryIndex(pathEntries, "program files (x86)\\dotnet"); + + if (x64Index < 0 && x86Index < 0) + { + return "no Windows dotnet installation directories were found in PATH"; + } + + if (x64Index >= 0 && x86Index < 0) + { + return "only x64 dotnet PATH entry found"; + } + + if (x64Index < 0 && x86Index >= 0) + { + return "only x86 dotnet PATH entry found"; + } + + return x86Index < x64Index + ? $"warning: x86 dotnet appears before x64 in PATH (x86 index {x86Index}, x64 index {x64Index})" + : $"x64 dotnet appears before x86 in PATH (x64 index {x64Index}, x86 index {x86Index})"; + } + + private static int FindPathEntryIndex(string[] pathEntries, string match) + { + for (int index = 0; index < pathEntries.Length; index++) + { + if (pathEntries[index].Contains(match, StringComparison.OrdinalIgnoreCase)) + { + return index; + } + } + + return -1; + } + + private static void PrintUsage() + { + Console.WriteLine(); + Console.WriteLine("Usage:"); + Console.WriteLine(" Issue3148PathProbe [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --connection-string, -c "); + Console.WriteLine(" SQL Server connection string for SqlClient.Open() test or stress test."); + Console.WriteLine(" If not provided, only environment information will be displayed."); + Console.WriteLine(); + Console.WriteLine(" --force-x86-dotnet-first"); + Console.WriteLine(" Prioritize x86 dotnet in PATH and re-launch probe in subprocess."); + Console.WriteLine(" Useful for testing x86 runtime behavior on x64 systems."); + Console.WriteLine(" Only works on Windows with both x86 and x64 dotnet installed."); + Console.WriteLine(); + Console.WriteLine(" --probe-sni-paths"); + Console.WriteLine(" Enumerate every location the runtime will probe for SNI.dll and report"); + Console.WriteLine(" existence and file size. Covers scenario A (DLL not deployed)."); + Console.WriteLine(" Windows-only. No connection string required."); + Console.WriteLine(); + Console.WriteLine(" --probe-arch-match"); + Console.WriteLine(" Read the PE header of each SNI.dll found on disk and compare its"); + Console.WriteLine(" machine type to the current process architecture."); + Console.WriteLine(" Reports MATCH or MISMATCH for each copy. Covers scenario B (arch mismatch)."); + Console.WriteLine(" Windows-only. No connection string required."); + Console.WriteLine(); + Console.WriteLine(" --probe-native-load"); + Console.WriteLine(" Call NativeLibrary.TryLoad() against each SNI.dll found on disk."); + Console.WriteLine(" A file that exists but fails to load indicates a missing VC++ dependency."); + Console.WriteLine(" Reports the Win32 error code on failure. Covers scenario C (missing deps)."); + Console.WriteLine(" Windows-only. No connection string required."); + Console.WriteLine(); + Console.WriteLine(" --probe-extraction-race"); + Console.WriteLine(" Delete extracted SNI.dll copies under %TEMP% then call SqlConnection.Open()."); + Console.WriteLine(" Reproduces scenario D (single-file extraction race / temp cleanup)."); + Console.WriteLine(" Only meaningful when run from a single-file published exe."); + Console.WriteLine(" Windows-only. Requires --connection-string."); + Console.WriteLine(); + Console.WriteLine(" --probe-lazy-load"); + Console.WriteLine(" Hold the process idle, then perform the first SqlConnection.Open()."); + Console.WriteLine(" Simulates long-running services where SNI is first loaded much later."); + Console.WriteLine(" Windows-only. Requires --connection-string."); + Console.WriteLine(); + Console.WriteLine(" --lazy-load-delay "); + Console.WriteLine(" Delay before first SqlConnection.Open() in --probe-lazy-load mode."); + Console.WriteLine(" Default: 60"); + Console.WriteLine(); + Console.WriteLine(" --lazy-load-disturb-sni"); + Console.WriteLine(" Disturb SNI load environment before first Open() by renaming every"); + Console.WriteLine(" discovered Microsoft.Data.SqlClient.SNI.dll file, then restoring after"); + Console.WriteLine(" the first Open() attempt completes."); + Console.WriteLine(" Only applies when --probe-lazy-load is used."); + Console.WriteLine(); + Console.WriteLine(" --stress-load-absent"); + Console.WriteLine(" Load-time absent probe (child-process mode)."); + Console.WriteLine(" Proves that error 0x8007007E is elicited when SNI is absent at LoadLibrary time."); + Console.WriteLine(" Spawns fresh child processes so SNI is loaded from scratch each time."); + Console.WriteLine(" Parent renames the SNI DLL before each child starts to create a genuine absent-at-load window."); + Console.WriteLine(" Requires --connection-string and only supported on Windows."); + Console.WriteLine(); + Console.WriteLine(" --stress-duration "); + Console.WriteLine(" Duration of stress test in seconds. Default: 30"); + Console.WriteLine(" Only applies when --stress-load-absent is used."); + Console.WriteLine(); + Console.WriteLine(" -h, --help, /?"); + Console.WriteLine(" Display this help message."); + Console.WriteLine(); + Console.WriteLine("Examples:"); + Console.WriteLine(" # Display environment information only"); + Console.WriteLine(" Issue3148PathProbe"); + Console.WriteLine(); + Console.WriteLine(" # Test SQL connection"); + Console.WriteLine(" Issue3148PathProbe --connection-string \"Server=localhost;User ID=sa;Password=xxx\""); + Console.WriteLine(); + Console.WriteLine(" # Enumerate SNI search paths"); + Console.WriteLine(" Issue3148PathProbe --probe-sni-paths"); + Console.WriteLine(); + Console.WriteLine(" # Check SNI architecture matches process"); + Console.WriteLine(" Issue3148PathProbe --probe-arch-match"); + Console.WriteLine(); + Console.WriteLine(" # Check VC++ dependencies load correctly"); + Console.WriteLine(" Issue3148PathProbe --probe-native-load"); + Console.WriteLine(); + Console.WriteLine(" # Single-file extraction race (run from published exe)"); + Console.WriteLine(" Issue3148PathProbe --probe-extraction-race --connection-string \"...\""); + Console.WriteLine(); + Console.WriteLine(" # Lazy first-load repro (delay then first Open)"); + Console.WriteLine(" Issue3148PathProbe --probe-lazy-load --lazy-load-delay 600 --connection-string \"...\""); + Console.WriteLine(); + Console.WriteLine(" # Lazy first-load repro with intentional disturbance"); + Console.WriteLine(" Issue3148PathProbe --probe-lazy-load --lazy-load-delay 30 --lazy-load-disturb-sni --connection-string \"...\""); + Console.WriteLine(); + Console.WriteLine(" # Load-time absent probe for 60 seconds"); + Console.WriteLine(" Issue3148PathProbe --stress-load-absent --connection-string \"...\" --stress-duration 60"); + Console.WriteLine(); + Console.WriteLine(" # Force x86 dotnet and test"); + Console.WriteLine(" Issue3148PathProbe --force-x86-dotnet-first --connection-string \"...\""); + Console.WriteLine(); + } + + private static int RunWithForcedDotnetPathOrdering(Options options) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--force-x86-dotnet-first is only supported on Windows."); + return 2; + } + + if (string.Equals(Environment.GetEnvironmentVariable(ChildModeEnvironmentVariable), "1", StringComparison.Ordinal)) + { + Console.WriteLine("Child re-exec already active; skipping another forced relaunch."); + PrintEnvironmentReport(); + return RunConnectionTest(options.ConnectionString); + } + + string? originalPath = Environment.GetEnvironmentVariable("PATH"); + string reorderedPath = BuildForcedDotnetPath(originalPath); + string launcherPath = ResolveCommandFromPath(GetDotnetCommandName(), reorderedPath) + ?? throw new InvalidOperationException("Unable to resolve dotnet from the reordered PATH."); + + string assemblyPath = Environment.ProcessPath ?? throw new InvalidOperationException("Cannot determine the current process path."); + if (!assemblyPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) + { + Console.Error.WriteLine("--force-x86-dotnet-first requires a framework-dependent launch via 'dotnet .dll'."); + Console.Error.WriteLine("Run the probe with 'dotnet run' or execute the built DLL through dotnet on Windows."); + return 2; + } + + string arguments = BuildChildArguments(options); + + Console.WriteLine("Launching child process with x86 dotnet PATH ordering..."); + Console.WriteLine($"Child launcher: {launcherPath}"); + + ProcessStartInfo startInfo = new() + { + FileName = launcherPath, + Arguments = $"\"{assemblyPath}\"{arguments}", + UseShellExecute = false + }; + + startInfo.Environment["PATH"] = reorderedPath; + startInfo.Environment[ChildModeEnvironmentVariable] = "1"; + + using Process child = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start the child process."); + child.WaitForExit(); + return child.ExitCode; + } + + // ------------------------------------------------------------------------- + // Probe A: enumerate every candidate SNI path and report existence + size + // ------------------------------------------------------------------------- + private static int RunProbeSniPaths() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--probe-sni-paths is only supported on Windows."); + return 2; + } + + Console.WriteLine("Probe: SNI path survey"); + Console.WriteLine(new string('=', 22)); + Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}"); + Console.WriteLine(); + + var paths = FindSniDllPaths().ToList(); + + if (paths.Count == 0) + { + Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found in any search location."); + Console.WriteLine(); + Console.WriteLine("Search roots checked:"); + foreach (string root in GetSniSearchRoots()) + { + Console.WriteLine($" {root} (exists={Directory.Exists(root)})"); + } + return 1; + } + + foreach (string dll in paths) + { + long size = new FileInfo(dll).Length; + Console.WriteLine($"[FOUND] {dll}"); + Console.WriteLine($" size={size} bytes"); + } + + Console.WriteLine(); + return 0; + } + + // ------------------------------------------------------------------------- + // Probe B: PE header inspection — architecture match + // ------------------------------------------------------------------------- + private static int RunProbeArchMatch() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--probe-arch-match is only supported on Windows."); + return 2; + } + + Console.WriteLine("Probe: SNI architecture match"); + Console.WriteLine(new string('=', 29)); + + Architecture processArch = RuntimeInformation.ProcessArchitecture; + Console.WriteLine($"Process architecture : {processArch}"); + Console.WriteLine(); + + var paths = FindSniDllPaths().ToList(); + if (paths.Count == 0) + { + Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found — cannot check architecture."); + return 1; + } + + int exitCode = 0; + foreach (string dll in paths) + { + Machine? machine = ReadPeMachine(dll); + if (machine is null) + { + Console.WriteLine($"[ERROR ] {dll}"); + Console.WriteLine($" Could not read PE header."); + exitCode = 1; + continue; + } + + string dllArch = machine.Value switch + { + Machine.I386 => "X86", + Machine.Amd64 => "X64", + Machine.Arm => "Arm", + Machine.Arm64 => "Arm64", + _ => $"unknown (0x{(ushort)machine.Value:X4})" + }; + + bool match = processArch switch + { + Architecture.X86 => machine.Value == Machine.I386, + Architecture.X64 => machine.Value == Machine.Amd64, + Architecture.Arm => machine.Value == Machine.Arm, + Architecture.Arm64 => machine.Value == Machine.Arm64, + _ => false + }; + + string verdict = match ? "[MATCH ]" : "[MISMATCH ]"; + Console.WriteLine($"{verdict} {dll}"); + Console.WriteLine($" DLL arch={dllArch} process arch={processArch}"); + + if (!match) + { + exitCode = 1; + } + } + + Console.WriteLine(); + return exitCode; + } + + private static Machine? ReadPeMachine(string dllPath) + { + try + { + using FileStream fs = File.OpenRead(dllPath); + using PEReader pe = new(fs); + return pe.PEHeaders.CoffHeader.Machine; + } + catch + { + return null; + } + } + + // ------------------------------------------------------------------------- + // Probe C: NativeLibrary.TryLoad — surfaces missing VC++ dependencies + // ------------------------------------------------------------------------- + private static int RunProbeNativeLoad() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--probe-native-load is only supported on Windows."); + return 2; + } + + Console.WriteLine("Probe: NativeLibrary.TryLoad"); + Console.WriteLine(new string('=', 27)); + Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}"); + Console.WriteLine(); + + var paths = FindSniDllPaths().ToList(); + if (paths.Count == 0) + { + Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found — nothing to load."); + return 1; + } + + int exitCode = 0; + foreach (string dll in paths) + { + bool loaded = NativeLibrary.TryLoad(dll, out IntPtr handle); + if (loaded) + { + Console.WriteLine($"[OK ] {dll}"); + NativeLibrary.Free(handle); + } + else + { + // NativeLibrary.TryLoad routes through managed exception handling, so the + // raw Win32 error is not accessible via GetLastWin32Error(). Diagnose the + // failure using our own PE header inspection instead. + Machine? machine = ReadPeMachine(dll); + Architecture processArch = RuntimeInformation.ProcessArchitecture; + bool archMismatch = machine is not null && !(processArch switch + { + Architecture.X86 => machine.Value == Machine.I386, + Architecture.X64 => machine.Value == Machine.Amd64, + Architecture.Arm => machine.Value == Machine.Arm, + Architecture.Arm64 => machine.Value == Machine.Arm64, + _ => false + }); + + Console.WriteLine($"[FAIL ] {dll}"); + if (archMismatch) + { + string dllArch = machine!.Value switch + { + Machine.I386 => "X86", + Machine.Amd64 => "X64", + Machine.Arm => "Arm", + Machine.Arm64 => "Arm64", + _ => $"0x{(ushort)machine.Value:X4}" + }; + Console.WriteLine($" Architecture mismatch: DLL={dllArch}, process={processArch}."); + Console.WriteLine($" Windows will report ERROR_BAD_EXE_FORMAT (193 / 0xC1) to the CLR."); + Console.WriteLine($" Use --probe-arch-match for a full arch survey."); + } + else + { + Console.WriteLine($" DLL exists and arch looks correct but load failed."); + Console.WriteLine($" Likely cause: a VC++ or other import-table dependency is missing."); + Console.WriteLine($" Windows returns ERROR_MOD_NOT_FOUND (126 / 0x7E) to the CLR."); + Console.WriteLine($" Install the Visual C++ redistributable matching the SNI DLL version."); + } + exitCode = 1; + } + } + + Console.WriteLine(); + return exitCode; + } + + // ------------------------------------------------------------------------- + // Probe D: single-file extraction race — delete extracted SNI then Open() + // ------------------------------------------------------------------------- + private static int RunProbeExtractionRace(Options options) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--probe-extraction-race is only supported on Windows."); + return 2; + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + Console.Error.WriteLine("--probe-extraction-race requires --connection-string."); + return 2; + } + + Console.WriteLine("Probe: single-file extraction race"); + Console.WriteLine(new string('=', 34)); + Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}"); + Console.WriteLine(); + + // Find SNI DLLs under %TEMP% only — those are single-file extraction candidates. + string tempRoot = Path.GetTempPath(); + List extracted = []; + try + { + extracted = Directory.GetFiles(tempRoot, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories).ToList(); + } + catch (Exception ex) + { + Console.WriteLine($"Could not enumerate temp directory: {ex.Message}"); + } + + if (extracted.Count == 0) + { + Console.WriteLine("No extracted SNI DLL found under %TEMP%."); + Console.WriteLine("This probe is only meaningful for single-file published apps."); + Console.WriteLine("Publish with -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true,"); + Console.WriteLine("run the published exe once so the runtime extracts SNI to %TEMP%,"); + Console.WriteLine("then re-run this probe to delete the extracted copy before SqlConnection.Open()."); + return 2; + } + + // Delete the extracted copies. This process has never loaded SNI, so no sharing violation. + List deleted = []; + foreach (string dll in extracted) + { + try + { + File.Delete(dll); + deleted.Add(dll); + Console.WriteLine($"[DELETED] {dll}"); + } + catch (Exception ex) + { + Console.WriteLine($"[SKIP ] {dll} — {ex.Message}"); + } + } + + if (deleted.Count == 0) + { + Console.WriteLine("Could not delete any extracted SNI DLL (files may be in use by another process)."); + return 2; + } + + Console.WriteLine(); + Console.WriteLine($"Deleted {deleted.Count} extracted SNI DLL(s). Attempting SqlConnection.Open()..."); + Console.WriteLine("If this is a framework-dependent app the runtime will find SNI via the packs cache and succeed."); + Console.WriteLine("Run this probe from a single-file published exe to reproduce the actual race."); + Console.WriteLine(); + + try + { + using SqlConnection connection = new(options.ConnectionString); + connection.Open(); + Console.WriteLine("SqlConnection.Open() succeeded (SNI found via fallback search path)."); + return 0; + } + catch (Exception ex) + { + Console.WriteLine("SqlConnection.Open() failed — SNI absent at LoadLibrary time."); + PrintExceptionTree(ex); + return 1; + } + } + + // ------------------------------------------------------------------------- + // Probe E: delayed first load — long-running service simulation + // ------------------------------------------------------------------------- + private static int RunProbeLazyLoad(Options options) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--probe-lazy-load is only supported on Windows."); + return 2; + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + Console.Error.WriteLine("--probe-lazy-load requires --connection-string."); + return 2; + } + + int delaySeconds = options.LazyLoadDelaySeconds; + if (delaySeconds < 0) + { + Console.Error.WriteLine("--lazy-load-delay must be >= 0."); + return 2; + } + + PrintEnvironmentReport(); + Console.WriteLine(); + Console.WriteLine("Probe: delayed first SNI load"); + Console.WriteLine(new string('=', 30)); + Console.WriteLine($"PID: {Environment.ProcessId}"); + Console.WriteLine($"Delay before first SqlConnection.Open(): {delaySeconds} seconds"); + Console.WriteLine("During this delay, you can simulate environment drift (temp cleanup, deploy changes, AV actions)."); + Console.WriteLine(); + + DateTime start = DateTime.UtcNow; + DateTime end = start.AddSeconds(delaySeconds); + + while (DateTime.UtcNow < end) + { + int remaining = (int)Math.Ceiling((end - DateTime.UtcNow).TotalSeconds); + Console.WriteLine($"Waiting... {remaining}s remaining"); + System.Threading.Thread.Sleep(1000); + } + + List<(string Original, string Temp)> disturbed = []; + if (options.LazyLoadDisturbSni) + { + Console.WriteLine(); + Console.WriteLine("Disturbance enabled: renaming discovered SNI DLL files before first Open()..."); + disturbed = RenameAllSniDllsForLazyLoad(".lazy_disturb"); + if (disturbed.Count == 0) + { + Console.WriteLine("No SNI files were renamed. Disturbance may have had no effect."); + } + else + { + Console.WriteLine($"Renamed {disturbed.Count} SNI file(s). First Open() will run while they are absent."); + } + } + + Console.WriteLine(); + Console.WriteLine("Delay complete. Attempting first SqlConnection.Open() now..."); + + int exitCode; + try + { + exitCode = RunConnectionTest(options.ConnectionString); + } + finally + { + if (disturbed.Count > 0) + { + RestoreSniDlls(disturbed); + Console.WriteLine(); + Console.WriteLine($"Restored {disturbed.Count} disturbed SNI file(s)."); + } + } + + return exitCode; + } + + private static int RunConnectionTest(string? connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + Console.WriteLine(); + Console.WriteLine("No connection string provided. Skipping SqlClient open test."); + return 0; + } + + Console.WriteLine(); + Console.WriteLine("Attempting SqlConnection.Open()..."); + + try + { + using SqlConnection connection = new(connectionString); + connection.Open(); + Console.WriteLine("SqlConnection.Open() succeeded."); + return 0; + } + catch (Exception ex) + { + Console.WriteLine("SqlConnection.Open() failed."); + PrintExceptionTree(ex); + return 1; + } + } + + private static string BuildForcedDotnetPath(string? originalPath) + { + string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":"; + List entries = (originalPath ?? string.Empty) + .Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToList(); + + List dotnetEntries = entries + .Where(entry => entry.Contains("program files", StringComparison.OrdinalIgnoreCase) + && entry.Contains("dotnet", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + string? x86Entry = dotnetEntries.FirstOrDefault(entry => entry.Contains("program files (x86)", StringComparison.OrdinalIgnoreCase)); + string? x64Entry = dotnetEntries.FirstOrDefault(entry => entry.Contains("program files\\dotnet", StringComparison.OrdinalIgnoreCase) + || entry.Contains("program files/dotnet", StringComparison.OrdinalIgnoreCase)); + + if (x86Entry is null || x64Entry is null) + { + throw new InvalidOperationException("Both x86 and x64 dotnet PATH entries are required to force the ordering."); + } + + entries.RemoveAll(entry => string.Equals(entry, x86Entry, StringComparison.OrdinalIgnoreCase) + || string.Equals(entry, x64Entry, StringComparison.OrdinalIgnoreCase)); + + entries.Insert(0, x64Entry); + entries.Insert(0, x86Entry); + return string.Join(separator, entries); + } + + private static string BuildChildArguments(Options options) + { + List arguments = []; + + if (!string.IsNullOrWhiteSpace(options.ConnectionString)) + { + arguments.Add("--connection-string"); + arguments.Add(options.ConnectionString); + } + + return arguments.Count == 0 + ? string.Empty + : " " + string.Join(" ", arguments.Select(QuoteArgument)); + } + + private static string QuoteArgument(string value) + => $"\"{value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal)}\""; + + private static int RunStressTest(Options options) + { + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + Console.Error.WriteLine("--stress-load-absent requires --connection-string."); + return 2; + } + + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Console.Error.WriteLine("--stress-load-absent is only supported on Windows."); + return 2; + } + + // Find the assembly to re-launch as children. + // When running as `dotnet foo.dll`, Environment.ProcessPath is the dotnet host, not the + // assembly. Use Assembly.Location which always points to the .dll (empty in single-file). + string assemblyLocation = typeof(Program).Assembly.Location; + bool isDll = !string.IsNullOrEmpty(assemblyLocation) + && assemblyLocation.EndsWith(".dll", StringComparison.OrdinalIgnoreCase); + + string childTarget; + string launcher; + + if (isDll) + { + childTarget = assemblyLocation; + launcher = ResolveCommandFromPath(GetDotnetCommandName()) + ?? throw new InvalidOperationException("Cannot resolve dotnet executable from PATH."); + } + else + { + // Self-contained or single-file exe — the process itself is the launcher. + childTarget = Environment.ProcessPath + ?? throw new InvalidOperationException("Cannot determine the current process path."); + launcher = childTarget; + } + + PrintEnvironmentReport(); + Console.WriteLine(); + Console.WriteLine("Starting load-time absent probe (child-process mode)..."); + Console.WriteLine($"Duration: {options.StressDurationSeconds} seconds"); + Console.WriteLine("Each iteration spawns a fresh child process so SNI loads from scratch."); + Console.WriteLine("Parent renames SNI DLLs before each child starts; the child sees SNI absent at LoadLibrary time."); + Console.WriteLine(); + + DateTime endTime = DateTime.UtcNow.AddSeconds(options.StressDurationSeconds); + int successCount = 0; + int failureCount = 0; + int renameWindows = 0; + int iteration = 0; + + while (DateTime.UtcNow < endTime) + { + iteration++; + + // Rename SNI DLLs so they are temporarily unavailable when the child process starts. + // The parent can do this because it has never loaded SNI itself. + List<(string Original, string Temp)> renamed = RenameAllSniDlls(".stress_rename"); + if (renamed.Count > 0) + { + renameWindows++; + } + + try + { + ProcessStartInfo startInfo = new() + { + UseShellExecute = false, + // Redirect child output so it doesn't flood the parent console. + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + if (isDll) + { + startInfo.FileName = launcher; + startInfo.ArgumentList.Add(childTarget); + } + else + { + startInfo.FileName = launcher; + } + + startInfo.ArgumentList.Add("--connection-string"); + startInfo.ArgumentList.Add(options.ConnectionString); + + using Process child = Process.Start(startInfo) + ?? throw new InvalidOperationException("Failed to start child process."); + + // Drain stdout/stderr async to prevent deadlock if the pipe buffer fills. + var stdoutDrain = child.StandardOutput.ReadToEndAsync(); + var stderrDrain = child.StandardError.ReadToEndAsync(); + + // Hold the rename window for up to 2 seconds — long enough for the child to + // start the CLR and call LoadLibrary for SNI — then restore so that if SNI + // is not yet loaded, the child can still pick it up from disk. + System.Threading.Thread.Sleep(2000); + RestoreSniDlls(renamed); + renamed = []; + + child.WaitForExit(); + stdoutDrain.GetAwaiter().GetResult(); + stderrDrain.GetAwaiter().GetResult(); + + if (child.ExitCode == 0) + { + successCount++; + Console.WriteLine($"[OK] iteration={iteration}"); + } + else + { + failureCount++; + Console.WriteLine($"[FAIL] iteration={iteration} exitCode={child.ExitCode}"); + } + } + finally + { + // Always restore — even if Process.Start or WaitForExit throws. + RestoreSniDlls(renamed); + } + } + + Console.WriteLine(); + Console.WriteLine("Stress test complete."); + Console.WriteLine($"Child process launches: {iteration}"); + Console.WriteLine($" Successful: {successCount}"); + Console.WriteLine($" Failed: {failureCount}"); + Console.WriteLine($"Iterations with rename window: {renameWindows} of {iteration}"); + + return failureCount > 0 ? 1 : 0; + } + + // Returns every SNI DLL found in on-disk search locations (packs cache and temp extraction + // directories), renamed to a temporary name. The parent process — which has never loaded + // SqlClient — can rename these without hitting a sharing violation. + private static List<(string Original, string Temp)> RenameAllSniDlls(string suffix) + => RenameSniDlls(FindSniDllPaths(), suffix); + + private static List<(string Original, string Temp)> RenameAllSniDllsForLazyLoad(string suffix) + => RenameSniDlls(FindSniDllPathsForLazyLoadDisturbance(), suffix); + + private static List<(string Original, string Temp)> RenameSniDlls(IEnumerable dllPaths, string suffix) + { + List<(string, string)> renamed = []; + + foreach (string dll in dllPaths) + { + string temp = dll + suffix; + try + { + File.Move(dll, temp); + renamed.Add((dll, temp)); + } + catch (Exception) + { + // File in use (loaded by another process), inaccessible, or permission-denied — skip. + } + } + + return renamed; + } + + private static IEnumerable FindSniDllPathsForLazyLoadDisturbance() + { + HashSet discovered = new(StringComparer.OrdinalIgnoreCase); + + // Include the baseline roots used by other probes. + foreach (string dll in FindSniDllPaths()) + { + discovered.Add(dll); + } + + // Include direct app-local candidates where framework-dependent outputs place native assets. + string[] directCandidates = + [ + Path.Combine(AppContext.BaseDirectory, "Microsoft.Data.SqlClient.SNI.dll"), + Path.Combine(AppContext.BaseDirectory, "runtimes", "win-x64", "native", "Microsoft.Data.SqlClient.SNI.dll"), + Path.Combine(AppContext.BaseDirectory, "runtimes", "win-x86", "native", "Microsoft.Data.SqlClient.SNI.dll"), + Path.Combine(AppContext.BaseDirectory, "runtimes", "win-arm64", "native", "Microsoft.Data.SqlClient.SNI.dll") + ]; + + foreach (string candidate in directCandidates) + { + if (File.Exists(candidate)) + { + discovered.Add(candidate); + } + } + + // Include runtime-directed native search paths and common NuGet package roots. + foreach (string root in GetLazyLoadDisturbanceRoots()) + { + if (!Directory.Exists(root)) + { + continue; + } + + string[] files; + try + { + files = Directory.GetFiles(root, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories); + } + catch (Exception) + { + continue; + } + + foreach (string file in files) + { + discovered.Add(file); + } + } + + return discovered; + } + + private static IEnumerable GetLazyLoadDisturbanceRoots() + { + HashSet roots = new(StringComparer.OrdinalIgnoreCase) + { + AppContext.BaseDirectory, + Path.Combine(AppContext.BaseDirectory, "runtimes") + }; + + string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + if (!string.IsNullOrWhiteSpace(userProfile)) + { + roots.Add(Path.Combine(userProfile, ".nuget", "packages", "microsoft.data.sqlclient.sni.runtime")); + roots.Add(Path.Combine(userProfile, ".nuget", "packages", "microsoft.data.sqlclient.sni")); + } + + string? nativeSearchDirs = Environment.GetEnvironmentVariable("NATIVE_DLL_SEARCH_DIRECTORIES"); + if (!string.IsNullOrWhiteSpace(nativeSearchDirs)) + { + string[] entries = nativeSearchDirs + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (string entry in entries) + { + roots.Add(entry); + } + } + + return roots; + } + + private static void RestoreSniDlls(List<(string Original, string Temp)> renamed) + { + foreach ((string original, string temp) in renamed) + { + try + { + if (File.Exists(temp)) + { + File.Move(temp, original, overwrite: true); + } + } + catch (Exception) + { + // Best-effort; log nothing so as not to pollute stress output. + } + } + } + + private static string[] GetSniSearchRoots() => + [ + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "dotnet", "packs"), + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "dotnet", "packs"), + Path.GetTempPath() + ]; + + private static IEnumerable FindSniDllPaths() + { + foreach (string root in GetSniSearchRoots()) + { + if (!Directory.Exists(root)) + { + continue; + } + + string[] files; + try + { + files = Directory.GetFiles(root, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories); + } + catch (Exception) + { + continue; + } + + foreach (string file in files) + { + yield return file; + } + } + } + + private sealed class Options + { + public string? ConnectionString { get; private init; } + public bool ForceX86DotnetFirst { get; private init; } + public bool ProbeSniPaths { get; private init; } + public bool ProbeArchMatch { get; private init; } + public bool ProbeNativeLoad { get; private init; } + public bool ProbeExtractionRace { get; private init; } + public bool ProbeLazyLoad { get; private init; } + public int LazyLoadDelaySeconds { get; private init; } = 60; + public bool LazyLoadDisturbSni { get; private init; } + public bool StressLoadAbsent { get; private init; } + public int StressDurationSeconds { get; private init; } = 30; + + public static Options Parse(string[] args) + { + string? connectionString = null; + bool forceX86DotnetFirst = false; + bool probeSniPaths = false; + bool probeArchMatch = false; + bool probeNativeLoad = false; + bool probeExtractionRace = false; + bool probeLazyLoad = false; + int lazyLoadDelaySeconds = 60; + bool lazyLoadDisturbSni = false; + bool stressTestSni = false; + int stressDurationSeconds = 30; + + for (int i = 0; i < args.Length; i++) + { + switch (args[i]) + { + case "--connection-string": + case "-c": + if (i + 1 >= args.Length) + { + throw new ArgumentException("Missing value for --connection-string."); + } + + connectionString = args[++i]; + break; + + case "--force-x86-dotnet-first": + forceX86DotnetFirst = true; + break; + + case "--probe-sni-paths": + probeSniPaths = true; + break; + + case "--probe-arch-match": + probeArchMatch = true; + break; + + case "--probe-native-load": + probeNativeLoad = true; + break; + + case "--probe-extraction-race": + probeExtractionRace = true; + break; + + case "--probe-lazy-load": + probeLazyLoad = true; + break; + + case "--lazy-load-delay": + if (i + 1 >= args.Length || !int.TryParse(args[i + 1], out int lazyDelay)) + { + throw new ArgumentException("Invalid or missing value for --lazy-load-delay."); + } + + if (lazyDelay < 0) + { + throw new ArgumentException("--lazy-load-delay must be >= 0."); + } + + lazyLoadDelaySeconds = lazyDelay; + i++; + break; + + case "--lazy-load-disturb-sni": + lazyLoadDisturbSni = true; + break; + + case "--stress-load-absent": + stressTestSni = true; + break; + + case "--stress-duration": + if (i + 1 >= args.Length || !int.TryParse(args[i + 1], out int duration)) + { + throw new ArgumentException("Invalid or missing value for --stress-duration."); + } + + stressDurationSeconds = duration; + i++; + break; + + case "--help": + case "-h": + case "/?": + throw new ArgumentException("Help requested."); + + default: + throw new ArgumentException($"Unknown argument: {args[i]}"); + } + } + + return new Options + { + ConnectionString = connectionString, + ForceX86DotnetFirst = forceX86DotnetFirst, + ProbeSniPaths = probeSniPaths, + ProbeArchMatch = probeArchMatch, + ProbeNativeLoad = probeNativeLoad, + ProbeExtractionRace = probeExtractionRace, + ProbeLazyLoad = probeLazyLoad, + LazyLoadDelaySeconds = lazyLoadDelaySeconds, + LazyLoadDisturbSni = lazyLoadDisturbSni, + StressLoadAbsent = stressTestSni, + StressDurationSeconds = stressDurationSeconds + }; + } + } +} diff --git a/investigations/issue-3148/README.md b/investigations/issue-3148/README.md new file mode 100644 index 0000000000..c9c31d413b --- /dev/null +++ b/investigations/issue-3148/README.md @@ -0,0 +1,705 @@ +# Issue 3148 Investigation App + +This folder contains a small standalone diagnostic app for investigating the +`0x8007007E` `Microsoft.Data.SqlClient.SNI.dll` load failure discussed in +issue `#3148`. + +The app is aimed at the path-ordering hypothesis: + +- show the current process architecture +- show the runtime architecture +- show the `PATH` entries that contain `dotnet` +- resolve `dotnet` from `PATH` the same way process launch does +- optionally relaunch itself with x86 `dotnet` ordered before x64 `dotnet` on Windows +- optionally try `SqlConnection.Open()` if a connection string is provided + +## Build + +```bash +dotnet build investigations/issue-3148/Issue3148PathProbe.csproj +``` + +## Run diagnostics only + +```bash +dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj +``` + +## Run with a connection attempt + +```bash +dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \ + --connection-string "Server=...;Database=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True" +``` + +## Force Windows PATH ordering + +This mode is intended to actively test the hypothesis behind issue `#3148`. +On Windows, the probe starts a fresh child process with `Program Files (x86)\\dotnet` +placed before `Program Files\\dotnet` in `PATH`, then runs the same diagnostics and +optional connection test in that child. + +This mode must be run as a framework-dependent app through `dotnet`, for example via +`dotnet run` or `dotnet Issue3148PathProbe.dll`. It does not validate host selection for +published executables directly. + +If both `Program Files (x86)\\dotnet` and `Program Files\\dotnet` are not present in +`PATH`, the probe prints a diagnostic message and exits with code `2`. + +```bash +dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \ + --force-x86-dotnet-first +``` + +You can combine it with a connection attempt: + +```bash +dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \ + --force-x86-dotnet-first \ + --connection-string "Server=...;Database=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True" +``` + +## Test with x86 runtime only (no x86 SDK required) + +For validating runtime behavior, run the built DLL directly with x86 `dotnet`. +This avoids SDK resolution and therefore avoids `global.json` SDK pinning. + +### 1) Verify x86 runtime installation + +```powershell +$x86Dotnet = 'C:\Program Files (x86)\dotnet\dotnet.exe' +if (Test-Path $x86Dotnet) { + & $x86Dotnet --list-runtimes +} +``` + +Look for `Microsoft.NETCore.App 8.0.26` in the output. + +### 2) Ensure the probe DLL exists + +```powershell +$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll' +if (-not (Test-Path $dll)) { + dotnet build investigations/issue-3148/Issue3148PathProbe.csproj -v minimal +} +``` + +### 3) Run with x86 host and x86-first PATH ordering + +```powershell +$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll' +$env:PATH = 'C:\Program Files (x86)\dotnet;C:\Program Files\dotnet;' + $env:PATH +& 'C:\Program Files (x86)\dotnet\dotnet.exe' $dll +Write-Output "EXIT:$LASTEXITCODE" +``` + +Expected signals in output: + +- `Process architecture: X86` +- `Framework: .NET 8.0.26` +- `Current process path: C:\Program Files (x86)\dotnet\dotnet.exe` +- `PATH ordering assessment: warning: x86 dotnet appears before x64 in PATH ...` +- `EXIT:0` + +## Scenario probes + +The four probes below each target one of the root-cause scenarios (A–D) identified +in the runtime analysis. They require no SQL Server connection and run in seconds. + +### Probe A — SNI path survey (`--probe-sni-paths`) + +Enumerates every location the runtime will probe for `Microsoft.Data.SqlClient.SNI.dll` +and reports existence and file size. Covers scenario A (DLL not deployed). + +```powershell +$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll' +& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-sni-paths +``` + +Expected output on a healthy machine: + +``` +Probe: SNI path survey +====================== +Process architecture: X64 + +[FOUND] C:\Users\...\AppData\Local\Temp\.net\Issue3148PathProbe\\Microsoft.Data.SqlClient.SNI.dll + size=566832 bytes +``` + +Exit code 1 with `[ABSENT]` means no SNI DLL was found anywhere — a deployment +problem. + +### Probe B — Architecture match (`--probe-arch-match`) + +Reads the PE `IMAGE_FILE_MACHINE` field of every SNI DLL found on disk and compares +it to `RuntimeInformation.ProcessArchitecture`. Covers scenario B (arch mismatch). + +```powershell +& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-arch-match +``` + +Expected output: + +``` +Probe: SNI architecture match +============================= +Process architecture : X64 + +[MATCH ] ...\Microsoft.Data.SqlClient.SNI.dll + DLL arch=X64 process arch=X64 +``` + +A `[MISMATCH]` line means the DLL on disk is the wrong architecture for this +process. The .NET runtime will report `ERROR_BAD_EXE_FORMAT` (193), not +`ERROR_MOD_NOT_FOUND` (126). If an arch mismatch appears *together* with no +matching-arch copy at all, the runtime will instead report `0x8007007E`. + +### Probe C — Native load (`--probe-native-load`) + +Calls `NativeLibrary.TryLoad()` against each SNI DLL found on disk. If the file +exists and the arch matches but load still fails, a VC++ or other import-table +dependency is missing. Covers scenario C (missing VC++ runtime). + +```powershell +& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-native-load +``` + +Expected output: + +``` +Probe: NativeLibrary.TryLoad +=========================== +Process architecture: X64 + +[OK ] ...\Microsoft.Data.SqlClient.SNI.dll +``` + +A `[FAIL]` line with "Architecture mismatch" confirms scenario B. A `[FAIL]` line +with "VC++ or other import-table dependency is missing" points to scenario C — +install the Visual C++ redistributable that matches the SNI DLL version. + +### Probe D — Single-file extraction race (`--probe-extraction-race`) + +Deletes extracted SNI DLL copies under `%TEMP%` then calls `SqlConnection.Open()`. +Simulates what happens when a temp-cleanup tool removes the extracted copy between +process startup and the first database connection. Covers scenario D (single-file +extraction race). + +This probe is only meaningful when run from a **single-file published exe** (see +`Publish a Windows single-file probe` below). When run as a framework-dependent +app, the runtime finds SNI via the packs cache and succeeds regardless. + +```powershell +# Publish first: +dotnet publish investigations/issue-3148/Issue3148PathProbe.csproj ` + -c Release -r win-x64 ` + -p:PublishSingleFile=true ` + -p:IncludeNativeLibrariesForSelfExtract=true + +# Run the published exe once so SNI is extracted to %TEMP%: +.\investigations\issue-3148\bin\Release\net8.0\win-x64\publish\Issue3148PathProbe.exe + +# Now run the probe — it deletes the extracted copy then calls Open(): +$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;' +.\investigations\issue-3148\bin\Release\net8.0\win-x64\publish\Issue3148PathProbe.exe ` + --probe-extraction-race --connection-string $cs +``` + +Expected output when the race is reproduced (no fallback copy exists): + +``` +[DELETED] C:\Users\...\AppData\Local\Temp\.net\Issue3148PathProbe\\Microsoft.Data.SqlClient.SNI.dll +... +SqlConnection.Open() failed — SNI absent at LoadLibrary time. +[0] System.DllNotFoundException: Unable to load DLL 'Microsoft.Data.SqlClient.SNI.dll'... +``` + +### Probe E — Delayed first load (`--probe-lazy-load`) + +This mode simulates a long-running process that does not touch SqlClient until +later. The probe waits for a configurable delay and only then performs the first +`SqlConnection.Open()`, which is when SNI is actually loaded. + +Use this mode when you need to reproduce "service ran for hours, then first DB +call failed with `0x8007007E`" behavior. + +```powershell +$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll' +$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;' +& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-lazy-load --lazy-load-delay 600 --connection-string $cs +``` + +To intentionally disturb the SNI load environment during the delay window: + +```powershell +& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-lazy-load --lazy-load-delay 30 --lazy-load-disturb-sni --connection-string $cs +``` + +Options for this probe: + +- `--probe-lazy-load` — Enable delayed first-load probe. +- `--lazy-load-delay ` — Delay before first `SqlConnection.Open()` (default: 60). +- `--lazy-load-disturb-sni` — Rename discovered `Microsoft.Data.SqlClient.SNI.dll` files (app output, native search directories, and relevant NuGet package roots) just before first `Open()`, then restore them after the attempt. +- `--connection-string` — Required. + +Expected output: + +``` +Probe: delayed first SNI load +============================== +PID: 12345 +Delay before first SqlConnection.Open(): 600 seconds +Waiting... 600s remaining +... +Delay complete. Attempting first SqlConnection.Open() now... +``` + +With `--lazy-load-disturb-sni`, output will also show how many SNI DLL files were +renamed before first `Open()`, and confirm they were restored afterward. + +If the environment has drifted during the wait (for example temp cleanup, +deployment changes, or AV timing), this first-open call is where `0x8007007E` +will surface. + +## Load-time absent probe (`--stress-load-absent`) + +This mode proves that error `0x8007007E` is elicited when SNI is absent at +`LoadLibrary` time. Because proof #1 shows that a process which has already loaded +SNI cannot encounter this error again, this mode tests only the **load-time path** +— it does not reproduce the long-running service scenario. + +### Why in-process connection loops cannot trigger the issue + +Once `Microsoft.Data.SqlClient.SNI.dll` is loaded via `LoadLibrary`, Windows holds +a lock on the file for the **lifetime of the process**. All subsequent +`SqlConnection.Open()` calls reuse the already-loaded native library without +touching the disk. An in-process loop therefore exercises the *post-load* path, +which is bulletproof by design. Rename attempts against an already-loaded DLL +always fail with a sharing-violation error. + +### How the child-process approach works + +The stress test spawns a **fresh child process** for each attempt. Each child goes +through the full `LoadLibrary` path from scratch. The parent process — which never +loads `SqlClient` or `SNI` itself — can rename the SNI DLL freely before the child +starts, creating a genuine load-time interference window. + +| Aspect | In-process loop (old) | Child-process spawn (new) | +|--------|----------------------|--------------------------| +| Connection attempts | Same process (SNI already loaded) | Fresh child process each time (SNI not yet loaded) | +| DLL rename window | After SNI loaded → always blocked (file in use) | Before child starts → succeeds every iteration | +| What is tested | `Open()`/`Close()` throughput | Actual `LoadLibrary` call path | +| Can trigger issue #3148? | No | Yes — if SNI is absent or wrong-arch when child calls `LoadLibrary` | + +The parent holds the rename for 2 seconds (long enough for the child CLR to start +and reach `LoadLibrary`), then restores the DLL so the child can still pick it up +if it hasn't loaded yet. This creates a race window matching the original failure +scenario. + +### Run the stress test + +```powershell +$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll' +$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;' +& 'C:\Program Files\dotnet\dotnet.exe' $dll --stress-load-absent --connection-string $cs --stress-duration 60 +``` + +Options: + +- `--stress-load-absent` — Enable load-time absent probe. Windows-only. +- `--connection-string` — Required. SQL Server connection string passed to each child process. +- `--stress-duration ` — How long to keep spawning children (default: 30 seconds). + +Expected output: + +``` +Starting SNI stress test (child-process mode)... +Duration: 60 seconds +Each iteration spawns a fresh child process so SNI loads from scratch. +Parent briefly renames SNI DLLs before each child starts to inject load-time interference. + +[OK] iteration=1 +[OK] iteration=2 +... + +Stress test complete. +Child process launches: 20 + Successful: 20 + Failed: 0 +Iterations with rename window: 20 of 20 +``` + +Interpreting results: + +- `Iterations with rename window: N of N` — confirms the parent found the SNI DLL on + disk and renamed it before each child. If this is `0 of N`, the DLL was not found + in the expected locations (check the packs cache or temp directory). +- `[OK]` — child loaded SNI and connected successfully. +- `[FAIL] exitCode=1` — child failed. Capturing the child's output (remove + `RedirectStandardOutput` temporarily) will show the full exception chain. + +A failing environment (wrong SNI architecture, AV interference, or a single-file +extraction race) would produce `[FAIL]` entries here. A clean lab environment with +the correct x64 SNI installed will typically show all `[OK]`. + +## Troubleshooting tips + +- `dotnet run` requires an SDK and is affected by `global.json`. +- `dotnet .dll` uses runtime resolution and is the preferred path for runtime-only testing. +- If `--force-x86-dotnet-first` prints that both x86 and x64 entries are required, add both folders to `PATH` for that session before running. +- If you need to test SDK commands under x86 host selection, install an x86 SDK version compatible with the repository `global.json`. +- The `--stress-test-sni` mode is Windows-only and requires a valid SQL connection string. +- If `Iterations with rename window: 0 of N`, the SNI DLL was not found in the packs cache + or temp directories. Run the probe once without `--stress-test-sni` to do a connection test + first; the NuGet package extraction will place the DLL in a temp folder that the stress + test can then locate. + + +## Publish a Windows single-file probe + +This is useful if you want to mirror the original report more closely. + +```bash +dotnet publish investigations/issue-3148/Issue3148PathProbe.csproj \ + -c Release \ + -r win-x64 \ + -p:PublishSingleFile=true \ + -p:IncludeNativeLibrariesForSelfExtract=true +``` + +## What to look for + +- If the first `dotnet` resolved from `PATH` is under `Program Files (x86)`, + that supports the path-ordering suspicion. +- If `--force-x86-dotnet-first` reproduces the failure in the child process, + that is strong evidence that launcher/runtime selection is part of the issue. +- If the process architecture and runtime architecture do not match the intended + deployment, that is another strong signal. +- If `SqlConnection.Open()` fails, the app prints the full exception chain so + the loader error code can be captured. + +--- + +## Background: Runtime source analysis + +This section documents findings from reading the .NET 8 runtime source code +(`v8.0.0` tag on `dotnet/runtime`). The evidence below informs both the design +of the stress test and the list of root causes to investigate on affected systems. + +### Proof: SNI.dll is loaded exactly once per process, never reloaded + +#### Entry point: `NDirectImportThunk` → `NDirectLink` (dllimport.cpp) + +The first time a P/Invoke method is called, control passes through +`NDirectImportThunk`, which calls `NDirectLink`. That function calls +`NativeLibrary::LoadLibraryFromMethodDesc`, obtains the `HMODULE`, resolves the +entry point, and then stores both permanently in the `NDirectMethodDesc`: + +```cpp +// dllimport.cpp — NDirectLink() +NATIVE_LIBRARY_HANDLE hmod = NativeLibrary::LoadLibraryFromMethodDesc(pMD); +// ... +LPVOID pvTarget = NDirectGetEntryPoint(pMD, hmod); +if (pvTarget) +{ + pMD->SetNDirectTarget(pvTarget); // ← written into NDirectMethodDesc once + fSuccess = TRUE; +} +``` + +`SetNDirectTarget` writes directly into `NDirectMethodDesc::m_pWriteableData->m_pNDirectTarget`. +Every subsequent call to that P/Invoke method jumps directly to that pointer. +`NDirectImportThunk` — and therefore `LoadLibrary` — is **never invoked again** +for that method. + +#### The `UnmanagedImageCache`: add-only, process-lifetime (nativelibrary.cpp + appdomain.cpp) + +Inside `LoadLibraryFromMethodDesc`, the call chain reaches `LoadNativeLibrary()`: + +```cpp +// nativelibrary.cpp — LoadNativeLibrary() +hmod = pDomain->FindUnmanagedImageInCache(wszLibName); // check cache first +if (hmod != NULL) + return hmod.Extract(); // ← return immediately, no LoadLibrary call + +hmod = LoadNativeLibraryBySearch(pMD, pErrorTracker, wszLibName); +if (hmod != NULL) +{ + pDomain->AddUnmanagedImageToCache(wszLibName, hmod); // ← store permanently + return hmod.Extract(); +} +``` + +The cache is implemented in `AppDomain::AddUnmanagedImageToCache()` (appdomain.cpp). +It allocates the key on the loader heap (which lives for the AppDomain lifetime) +and inserts into a hash table. There is **no `RemoveUnmanagedImageFromCache`** +anywhere in the codebase. The cache is insert-only. + +In .NET 8, there is a single AppDomain that lasts for the entire process lifetime. +Therefore: + +- `LoadLibrary` is called for `SNI.dll` **at most once per process**. +- After the first successful load the HMODULE is cached; all future P/Invoke + calls use the cached function pointer and never touch the disk. +- `FreeLibrary` is never called for P/Invoke-loaded libraries during normal + process execution. The OS keeps the DLL mapped until process exit. + +#### Why this matters for a long-running service + +Because loading is **lazy** (triggered by the first `SqlConnection.Open()`, not at +process startup), the `LoadLibrary` call can happen at any point during the service's +lifetime — potentially hours after startup. Whether the load succeeds depends on +filesystem state, AV scanner activity, and environment **at that moment**. +Subsequent connections reuse the cached pointer and never touch the disk, which is +why affected services often recover after a process restart that re-loads SNI.dll +under favourable conditions. + +--- + +### All scenarios where error `0x8007007E` can occur + +`0x8007007E = HRESULT_FROM_WIN32(ERROR_MOD_NOT_FOUND)` — Win32 error 126, +"The specified module could not be found." + +The runtime converts this to `DllNotFoundException` inside +`LoadLibErrorTracker::Throw()` (nativelibrary.cpp): + +```cpp +HRESULT theHRESULT = GetHR(); +if (theHRESULT == HRESULT_FROM_WIN32(ERROR_BAD_EXE_FORMAT)) + COMPlusThrow(kBadImageFormatException); +else +{ + SString hrString; + GetHRMsg(theHRESULT, hrString); + COMPlusThrow(kDllNotFoundException, IDS_EE_NDIRECT_LOADLIB_WIN, + libraryNameOrPath.GetUnicode(), hrString); +} +``` + +The error tracker assigns priorities when multiple probes are attempted: + +```cpp +case ERROR_FILE_NOT_FOUND: +case ERROR_PATH_NOT_FOUND: +case ERROR_MOD_NOT_FOUND: // priority 10 — lowest +case ERROR_DLL_NOT_FOUND: + priority = const_priorityNotFound; + break; +case ERROR_ACCESS_DENIED: + priority = const_priorityAccessDenied; // priority 20 +default: + priority = const_priorityCouldNotLoad; // priority 99999 — highest +``` + +The tracker keeps only the highest-priority error. Seeing `0x8007007E` in the +exception means **every** search strategy returned "not found" or lower — no probe +hit a higher-priority error such as access denied or a corrupt image. + +#### Search order that must exhaust before the exception is thrown + +`LoadNativeLibraryBySearch()` probes these locations in order: + +1. Windows API sets (skipped — SNI is not an API set name) +2. Directories in `NATIVE_DLL_SEARCH_DIRECTORIES` (set by the host) +3. The P/Invoke assembly's own directory (the `.dll`/`.exe` folder) +4. `LoadLibraryEx` with `dllImportSearchPathFlags` (assembly-level or default) + +All of these must fail for `0x8007007E` to be reported. + +#### Scenario A — SNI DLL not deployed + +The file is simply absent from all probed paths. This is the most common root +cause when a deployment is incomplete (missing the +`runtimes/win-x64/native/` or `runtimes/win-x86/native/` subtree from the NuGet +layout). Results in a consistent, non-intermittent failure on every +`SqlConnection.Open()`. + +#### Scenario B — Architecture mismatch: no matching-arch DLL present + +A 32-bit service process on a machine that has only the x64 NuGet layout (or vice +versa). The runtime probes for a DLL whose bitness matches the process, finds +none, and reports `0x8007007E`. If the wrong-arch DLL is found and Windows tries +to load it into the mismatched process, `ERROR_BAD_EXE_FORMAT` (193) is returned +instead, producing `BadImageFormatException` rather than `DllNotFoundException`. +Getting `0x8007007E` with an arch mismatch therefore means there is no DLL at all +for that architecture — not even a wrong-arch one. + +This is the scenario most consistent with issue #3148's description: a Windows +service that runs as 32-bit (`x86`) on a machine whose deployment only includes +the `win-x64` NuGet runtime assets. + +#### Scenario C — A dependency of SNI.dll is missing + +`Microsoft.Data.SqlClient.SNI.dll` is a native C++ DLL with its own import table. +It depends on system DLLs and the Visual C++ runtime (`VCRUNTIME140.dll`, +`MSVCP140.dll`, etc.). When `LoadLibrary` maps SNI.dll, the Windows loader +recursively resolves its imports. If any **direct or transitive** dependency is +missing, the Windows loader returns `ERROR_MOD_NOT_FOUND` for the outer call to +`SNI.dll` — even though SNI.dll itself was found on disk. The missing dependency +is not identified in the exception message. + +Common in stripped server images, minimal Docker base images, or servers that +lack the Visual C++ redistributable. + +#### Scenario D — Single-file app extraction race + +When published with `PublishSingleFile=true` and +`IncludeNativeLibrariesForSelfExtract=true`, native DLLs are extracted to a +temp directory at startup (e.g., +`%TEMP%\.net\\\`). Extraction and `LoadLibrary` are two +distinct operations. If a cleanup tool, AV scanner, or another process removes +the extracted file in the window between extraction and the first +`SqlConnection.Open()`, the file is gone when `LoadLibrary` is called → +`ERROR_MOD_NOT_FOUND`. This is intermittent and can occur hours after startup +if the service's first database connection is delayed. + +#### Scenario E — AV/EDR scan-on-access interception + +Some endpoint security products intercept `LoadLibrary` calls by briefly renaming +or locking the target file during an on-access scan. From the calling process's +perspective the file disappears between the search pass and the map pass, and the +OS loader reports `ERROR_MOD_NOT_FOUND`. This failure is intermittent and +timing-dependent — it occurs only when the AV scanner's scan window overlaps with +the `LoadLibrary` call, which can happen at any point during the process lifetime +(including long after startup). + +This is the scenario the `--stress-load-absent` mode's rename-window mechanism is designed to +simulate. + +#### Scenario F — Directory-level ACL change after deployment + +If the runtime probes the assembly directory but the **directory** itself (not just +the DLL file) loses read/traverse permission after deployment, the probe silently +finds nothing and reports `ERROR_MOD_NOT_FOUND`. If the DLL file itself loses read +permission, `ERROR_ACCESS_DENIED` is reported instead (and would override +`0x8007007E` in the error tracker). The directory case therefore produces +`0x8007007E` even though the file physically exists. + +#### Scenario G — SxS/manifest activation failure (rare) + +`Microsoft.Data.SqlClient.SNI.dll` ships with an embedded side-by-side manifest +specifying a VC++ runtime version. If the activation context creation fails (e.g., +the required VC++ merge module is not installed on the system), Windows returns +`ERROR_MOD_NOT_FOUND` for the outer `LoadLibrary` call rather than a more specific +SxS error. This is uncommon in modern deployments but can occur on images built +with minimal Windows component selection. + +#### Summary + +| Scenario | Error consistent? | Intermittent? | Most relevant to issue #3148? | +|---|---|---|---| +| **A** — SNI DLL not deployed | Consistent | No | Unlikely (service ran before) | +| **B** — Arch mismatch, no matching-arch DLL | Consistent | No | **Yes — 32-bit service, x64-only deploy** | +| **C** — VC++ runtime dependency missing | Consistent | No | Possible on minimal images | +| **D** — Single-file extraction race | Intermittent | Yes | Yes — if using single-file publish | +| **E** — AV/EDR scan-on-access | Intermittent | Yes | **Yes — matches intermittent pattern** | +| **F** — Directory ACL change | Intermittent | Yes | Possible in hardened environments | +| **G** — SxS/manifest version mismatch | Consistent | No | Unlikely | + +Scenarios B, D, and E are the primary candidates for the "works after restart, +fails intermittently in a long-running service" pattern described in issue #3148. + +--- + +### Interpreting the long-running-process symptom + +Issue #3148 describes a process that runs for a long time, performs database work, +and only later reports `0x8007007E` for `Microsoft.Data.SqlClient.SNI.dll`. + +Given the runtime behavior above (native DLL load is cached per process and never +reloaded), a true "loaded successfully and then failed later in the same process" +sequence is not expected in normal operation. + +In practice, the symptom usually means one of the following: + +1. The first SNI load was delayed (lazy load), so the process had been running but + had not yet reached the first `SqlConnection.Open()` that triggers `LoadLibrary`. +2. The reported failure came from another process instance, worker, or recycle event, + while logs were interpreted as one continuous process. +3. A non-standard event occurred (for example process/host recycle, unusual loading + context transitions, or environmental interference around first load). + +#### What "something extraordinary happens" means + +In this context, "extraordinary" means an event that breaks the normal assumption +of one stable OS process with one successful first native load that remains cached. + +Concrete examples: + +1. **Silent process replacement (most common)** + - Service restart, app pool recycle, container restart, watchdog restart, crash-restart. + - Operationally appears as one continuous service, but it is a *new process* doing + a fresh first `LoadLibrary`. + - Evidence to collect: PID changes, startup timestamps, service control events, + container instance IDs. + +2. **"Successful earlier work" did not include SNI load yet** + - Service was active for hours, but first SqlClient path was delayed. + - Failure appears "late" only because first `SqlConnection.Open()` happened late. + - Evidence to collect: application timeline showing first DB call aligns with failure. + +3. **Failure occurred in a different worker process** + - Parent process appears healthy; child worker/sidecar process is the one failing + its first load. + - Evidence to collect: executable path + PID in logs for each failure record. + +4. **Host environment changed before a new first-load event** + - During recycle/deploy window, path/deployment/temp/AV state changed. + - New process then fails first load with `0x8007007E`. + - Evidence to collect: deployment history, temp cleanup tasks, AV quarantine logs, + file inventory before/after restart. + +5. **In-process native tampering (rare)** + - Injected tools/hooks alter loader behavior or call `FreeLibrary` unexpectedly. + - This can violate normal runtime invariants. + - Evidence to collect: unexpected native modules, endpoint instrumentation, + debugger/injector presence. + +6. **Severe memory corruption (very rare)** + - Unsafe/native corruption damages runtime or loader state. + - Not specific to SNI; usually accompanied by broader instability. + - Evidence to collect: access violations, Watson dumps, random unrelated faults. + +7. **Later code path loads an additional native dependency (edge case)** + - SNI may be present, but a dependency touched by a later feature path is missing. + - Can look like "worked earlier, failed later" if paths differ. + - Evidence to collect: feature-specific repro matrix; compare auth/query paths. + +#### Most likely root-cause pattern + +The strongest fit for "works for a while, then fails" is delayed first load plus +environment drift before the first database call: + +- Service starts and does non-database work. +- SNI is not loaded yet (lazy native load path). +- Temp cleanup, deployment drift, architecture mismatch exposure, or AV timing + changes the load environment. +- First connection attempt happens later and fails with `0x8007007E`. + +This pattern maps directly to scenarios B, D, and E: + +- **B**: no matching-architecture SNI DLL is available for the process bitness. +- **D**: single-file extracted native payload under `%TEMP%` is removed before first load. +- **E**: AV/EDR transiently interferes with the file at load time. + +#### How to reason about scenario C in this symptom pattern + +Scenario C (missing transitive dependency such as VC++ runtime) is still possible, +but it is usually consistent rather than intermittent. It appears "delayed" only +when the first connection is delayed. Once the process has loaded SNI successfully, +scenario C should not reappear inside that same process lifetime. + +#### Why this matters for reproductions + +Because SNI load is one-time per process, reproductions should focus on the first +load window, not steady-state `Open()/Close()` loops after successful load. + +That is why the tooling in this folder separates: + +- load-window probes (`--probe-*` modes), and +- load-time interference stress (`--stress-load-absent`). + +These modes are designed to diagnose first-load failures that surface as +`0x8007007E`, including cases that appear as "long-running" in operational logs. diff --git a/investigations/issue-3148/global.json b/investigations/issue-3148/global.json new file mode 100644 index 0000000000..32dd4ed338 --- /dev/null +++ b/investigations/issue-3148/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.203", + "rollForward": "disable", + "allowPrerelease": false + } +} \ No newline at end of file