From 7e3e1c256ea42614b44701115225eae600f14a24 Mon Sep 17 00:00:00 2001
From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com>
Date: Fri, 24 Apr 2026 09:43:48 -0300
Subject: [PATCH 1/2] Add issue 3148 path probe app
---
.../issue-3148/Issue3148PathProbe.csproj | 14 +
investigations/issue-3148/Program.cs | 364 ++++++++++++++++++
investigations/issue-3148/README.md | 80 ++++
investigations/issue-3148/global.json | 7 +
4 files changed, 465 insertions(+)
create mode 100644 investigations/issue-3148/Issue3148PathProbe.csproj
create mode 100644 investigations/issue-3148/Program.cs
create mode 100644 investigations/issue-3148/README.md
create mode 100644 investigations/issue-3148/global.json
diff --git a/investigations/issue-3148/Issue3148PathProbe.csproj b/investigations/issue-3148/Issue3148PathProbe.csproj
new file mode 100644
index 0000000000..20500a2697
--- /dev/null
+++ b/investigations/issue-3148/Issue3148PathProbe.csproj
@@ -0,0 +1,14 @@
+
+
+ Exe
+ net8.0
+ enable
+ enable
+ latest
+ false
+
+
+
+
+
+
\ No newline at end of file
diff --git a/investigations/issue-3148/Program.cs b/investigations/issue-3148/Program.cs
new file mode 100644
index 0000000000..64179daa5a
--- /dev/null
+++ b/investigations/issue-3148/Program.cs
@@ -0,0 +1,364 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using Microsoft.Data.SqlClient;
+
+internal static class Program
+{
+ private const string ChildModeEnvironmentVariable = "ISSUE3148_PATH_PROBE_CHILD";
+
+ private static int Main(string[] args)
+ {
+ Options options;
+
+ try
+ {
+ options = Options.Parse(args);
+ }
+ catch (ArgumentException ex)
+ {
+ Console.Error.WriteLine(ex.Message);
+ PrintUsage();
+ return 2;
+ }
+
+ if (options.ForceX86DotnetFirst)
+ {
+ return RunWithForcedDotnetPathOrdering(options);
+ }
+
+ PrintEnvironmentReport();
+
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ Console.WriteLine();
+ Console.WriteLine("No connection string provided. Skipping SqlClient open test.");
+ return 0;
+ }
+
+ Console.WriteLine();
+ Console.WriteLine("Attempting SqlConnection.Open()...");
+
+ try
+ {
+ using SqlConnection connection = new(options.ConnectionString);
+ connection.Open();
+ Console.WriteLine("SqlConnection.Open() succeeded.");
+ return 0;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("SqlConnection.Open() failed.");
+ PrintExceptionTree(ex);
+ return 1;
+ }
+ }
+
+ private static void PrintEnvironmentReport()
+ {
+ Console.WriteLine("Issue 3148 PATH probe");
+ Console.WriteLine(new string('=', 21));
+ Console.WriteLine($"OS description: {RuntimeInformation.OSDescription}");
+ Console.WriteLine($"OS architecture: {RuntimeInformation.OSArchitecture}");
+ Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}");
+ Console.WriteLine($"Framework: {RuntimeInformation.FrameworkDescription}");
+ Console.WriteLine($"Current process path: {Environment.ProcessPath ?? ""}");
+ Console.WriteLine($"Current directory: {Environment.CurrentDirectory}");
+ Console.WriteLine();
+
+ string? pathValue = Environment.GetEnvironmentVariable("PATH");
+ string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":";
+ string[] pathEntries = (pathValue ?? string.Empty)
+ .Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+
+ Console.WriteLine("PATH entries containing 'dotnet':");
+ int dotnetEntryCount = 0;
+
+ for (int index = 0; index < pathEntries.Length; index++)
+ {
+ if (pathEntries[index].Contains("dotnet", StringComparison.OrdinalIgnoreCase))
+ {
+ dotnetEntryCount++;
+ Console.WriteLine($" [{index}] {pathEntries[index]}");
+ }
+ }
+
+ if (dotnetEntryCount == 0)
+ {
+ Console.WriteLine(" ");
+ }
+
+ Console.WriteLine();
+ Console.WriteLine($"Resolved dotnet executable from PATH: {ResolveCommandFromPath(GetDotnetCommandName()) ?? ""}");
+ Console.WriteLine($"PATH ordering assessment: {AssessDotnetPathOrdering(pathEntries)}");
+ }
+
+ private static string GetDotnetCommandName()
+ => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dotnet.exe" : "dotnet";
+
+ private static string? ResolveCommandFromPath(string commandName)
+ {
+ string? pathValue = Environment.GetEnvironmentVariable("PATH");
+ return ResolveCommandFromPath(commandName, pathValue);
+ }
+
+ private static string? ResolveCommandFromPath(string commandName, string? pathValue)
+ {
+ if (string.IsNullOrWhiteSpace(pathValue))
+ {
+ return null;
+ }
+
+ string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":";
+ string[] pathEntries = pathValue.Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+
+ foreach (string entry in pathEntries)
+ {
+ try
+ {
+ string candidate = Path.Combine(entry, commandName);
+ if (File.Exists(candidate))
+ {
+ return candidate;
+ }
+ }
+ catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
+ {
+ }
+ }
+
+ return null;
+ }
+
+ private static void PrintExceptionTree(Exception ex)
+ {
+ int depth = 0;
+ Exception? current = ex;
+
+ while (current is not null)
+ {
+ Console.WriteLine($"[{depth}] {current.GetType().FullName}: {current.Message}");
+ Console.WriteLine(current.StackTrace);
+ Console.WriteLine();
+ current = current.InnerException;
+ depth++;
+ }
+ }
+
+ private static string AssessDotnetPathOrdering(string[] pathEntries)
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ return "non-Windows environment; x86/x64 Windows PATH ordering check not applicable";
+ }
+
+ int x64Index = FindPathEntryIndex(pathEntries, "program files\\dotnet");
+ int x86Index = FindPathEntryIndex(pathEntries, "program files (x86)\\dotnet");
+
+ if (x64Index < 0 && x86Index < 0)
+ {
+ return "no Windows dotnet installation directories were found in PATH";
+ }
+
+ if (x64Index >= 0 && x86Index < 0)
+ {
+ return "only x64 dotnet PATH entry found";
+ }
+
+ if (x64Index < 0 && x86Index >= 0)
+ {
+ return "only x86 dotnet PATH entry found";
+ }
+
+ return x86Index < x64Index
+ ? $"warning: x86 dotnet appears before x64 in PATH (x86 index {x86Index}, x64 index {x64Index})"
+ : $"x64 dotnet appears before x86 in PATH (x64 index {x64Index}, x86 index {x86Index})";
+ }
+
+ private static int FindPathEntryIndex(string[] pathEntries, string match)
+ {
+ for (int index = 0; index < pathEntries.Length; index++)
+ {
+ if (pathEntries[index].Contains(match, StringComparison.OrdinalIgnoreCase))
+ {
+ return index;
+ }
+ }
+
+ return -1;
+ }
+
+ private static void PrintUsage()
+ {
+ Console.WriteLine();
+ Console.WriteLine("Usage:");
+ Console.WriteLine(" Issue3148PathProbe [--connection-string ] [--force-x86-dotnet-first]");
+ }
+
+ private static int RunWithForcedDotnetPathOrdering(Options options)
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--force-x86-dotnet-first is only supported on Windows.");
+ return 2;
+ }
+
+ if (string.Equals(Environment.GetEnvironmentVariable(ChildModeEnvironmentVariable), "1", StringComparison.Ordinal))
+ {
+ Console.WriteLine("Child re-exec already active; skipping another forced relaunch.");
+ PrintEnvironmentReport();
+ return RunConnectionTest(options.ConnectionString);
+ }
+
+ string? originalPath = Environment.GetEnvironmentVariable("PATH");
+ string reorderedPath = BuildForcedDotnetPath(originalPath);
+ string launcherPath = ResolveCommandFromPath(GetDotnetCommandName(), reorderedPath)
+ ?? throw new InvalidOperationException("Unable to resolve dotnet from the reordered PATH.");
+
+ string assemblyPath = Environment.ProcessPath ?? throw new InvalidOperationException("Cannot determine the current process path.");
+ if (!assemblyPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase))
+ {
+ Console.Error.WriteLine("--force-x86-dotnet-first requires a framework-dependent launch via 'dotnet .dll'.");
+ Console.Error.WriteLine("Run the probe with 'dotnet run' or execute the built DLL through dotnet on Windows.");
+ return 2;
+ }
+
+ string arguments = BuildChildArguments(options);
+
+ Console.WriteLine("Launching child process with x86 dotnet PATH ordering...");
+ Console.WriteLine($"Child launcher: {launcherPath}");
+
+ ProcessStartInfo startInfo = new()
+ {
+ FileName = launcherPath,
+ Arguments = $"\"{assemblyPath}\"{arguments}",
+ UseShellExecute = false
+ };
+
+ startInfo.Environment["PATH"] = reorderedPath;
+ startInfo.Environment[ChildModeEnvironmentVariable] = "1";
+
+ using Process child = Process.Start(startInfo) ?? throw new InvalidOperationException("Failed to start the child process.");
+ child.WaitForExit();
+ return child.ExitCode;
+ }
+
+ private static int RunConnectionTest(string? connectionString)
+ {
+ if (string.IsNullOrWhiteSpace(connectionString))
+ {
+ Console.WriteLine();
+ Console.WriteLine("No connection string provided. Skipping SqlClient open test.");
+ return 0;
+ }
+
+ Console.WriteLine();
+ Console.WriteLine("Attempting SqlConnection.Open()...");
+
+ try
+ {
+ using SqlConnection connection = new(connectionString);
+ connection.Open();
+ Console.WriteLine("SqlConnection.Open() succeeded.");
+ return 0;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("SqlConnection.Open() failed.");
+ PrintExceptionTree(ex);
+ return 1;
+ }
+ }
+
+ private static string BuildForcedDotnetPath(string? originalPath)
+ {
+ string separator = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ";" : ":";
+ List entries = (originalPath ?? string.Empty)
+ .Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+ .ToList();
+
+ List dotnetEntries = entries
+ .Where(entry => entry.Contains("program files", StringComparison.OrdinalIgnoreCase)
+ && entry.Contains("dotnet", StringComparison.OrdinalIgnoreCase))
+ .ToList();
+
+ string? x86Entry = dotnetEntries.FirstOrDefault(entry => entry.Contains("program files (x86)", StringComparison.OrdinalIgnoreCase));
+ string? x64Entry = dotnetEntries.FirstOrDefault(entry => entry.Contains("program files\\dotnet", StringComparison.OrdinalIgnoreCase)
+ || entry.Contains("program files/dotnet", StringComparison.OrdinalIgnoreCase));
+
+ if (x86Entry is null || x64Entry is null)
+ {
+ throw new InvalidOperationException("Both x86 and x64 dotnet PATH entries are required to force the ordering.");
+ }
+
+ entries.RemoveAll(entry => string.Equals(entry, x86Entry, StringComparison.OrdinalIgnoreCase)
+ || string.Equals(entry, x64Entry, StringComparison.OrdinalIgnoreCase));
+
+ entries.Insert(0, x64Entry);
+ entries.Insert(0, x86Entry);
+ return string.Join(separator, entries);
+ }
+
+ private static string BuildChildArguments(Options options)
+ {
+ List arguments = [];
+
+ if (!string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ arguments.Add("--connection-string");
+ arguments.Add(options.ConnectionString);
+ }
+
+ return arguments.Count == 0
+ ? string.Empty
+ : " " + string.Join(" ", arguments.Select(QuoteArgument));
+ }
+
+ private static string QuoteArgument(string value)
+ => $"\"{value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
+
+ private sealed class Options
+ {
+ public string? ConnectionString { get; private init; }
+ public bool ForceX86DotnetFirst { get; private init; }
+
+ public static Options Parse(string[] args)
+ {
+ string? connectionString = null;
+ bool forceX86DotnetFirst = false;
+
+ for (int i = 0; i < args.Length; i++)
+ {
+ switch (args[i])
+ {
+ case "--connection-string":
+ case "-c":
+ if (i + 1 >= args.Length)
+ {
+ throw new ArgumentException("Missing value for --connection-string.");
+ }
+
+ connectionString = args[++i];
+ break;
+
+ case "--force-x86-dotnet-first":
+ forceX86DotnetFirst = true;
+ break;
+
+ case "--help":
+ case "-h":
+ case "/?":
+ throw new ArgumentException("Help requested.");
+
+ default:
+ throw new ArgumentException($"Unknown argument: {args[i]}");
+ }
+ }
+
+ return new Options
+ {
+ ConnectionString = connectionString,
+ ForceX86DotnetFirst = forceX86DotnetFirst
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/investigations/issue-3148/README.md b/investigations/issue-3148/README.md
new file mode 100644
index 0000000000..d40ca97262
--- /dev/null
+++ b/investigations/issue-3148/README.md
@@ -0,0 +1,80 @@
+# Issue 3148 Investigation App
+
+This folder contains a small standalone diagnostic app for investigating the
+`0x8007007E` `Microsoft.Data.SqlClient.SNI.dll` load failure discussed in
+issue `#3148`.
+
+The app is aimed at the path-ordering hypothesis:
+
+- show the current process architecture
+- show the runtime architecture
+- show the `PATH` entries that contain `dotnet`
+- resolve `dotnet` from `PATH` the same way process launch does
+- optionally relaunch itself with x86 `dotnet` ordered before x64 `dotnet` on Windows
+- optionally try `SqlConnection.Open()` if a connection string is provided
+
+## Build
+
+```bash
+dotnet build investigations/issue-3148/Issue3148PathProbe.csproj
+```
+
+## Run diagnostics only
+
+```bash
+dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj
+```
+
+## Run with a connection attempt
+
+```bash
+dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \
+ --connection-string "Server=...;Database=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True"
+```
+
+## Force Windows PATH ordering
+
+This mode is intended to actively test the hypothesis behind issue `#3148`.
+On Windows, the probe starts a fresh child process with `Program Files (x86)\\dotnet`
+placed before `Program Files\\dotnet` in `PATH`, then runs the same diagnostics and
+optional connection test in that child.
+
+This mode must be run as a framework-dependent app through `dotnet`, for example via
+`dotnet run` or `dotnet Issue3148PathProbe.dll`. It does not validate host selection for
+published executables directly.
+
+```bash
+dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \
+ --force-x86-dotnet-first
+```
+
+You can combine it with a connection attempt:
+
+```bash
+dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \
+ --force-x86-dotnet-first \
+ --connection-string "Server=...;Database=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True"
+```
+
+## Publish a Windows single-file probe
+
+This is useful if you want to mirror the original report more closely.
+
+```bash
+dotnet publish investigations/issue-3148/Issue3148PathProbe.csproj \
+ -c Release \
+ -r win-x64 \
+ -p:PublishSingleFile=true \
+ -p:IncludeNativeLibrariesForSelfExtract=true
+```
+
+## What to look for
+
+- If the first `dotnet` resolved from `PATH` is under `Program Files (x86)`,
+ that supports the path-ordering suspicion.
+- If `--force-x86-dotnet-first` reproduces the failure in the child process,
+ that is strong evidence that launcher/runtime selection is part of the issue.
+- If the process architecture and runtime architecture do not match the intended
+ deployment, that is another strong signal.
+- If `SqlConnection.Open()` fails, the app prints the full exception chain so
+ the loader error code can be captured.
\ No newline at end of file
diff --git a/investigations/issue-3148/global.json b/investigations/issue-3148/global.json
new file mode 100644
index 0000000000..32dd4ed338
--- /dev/null
+++ b/investigations/issue-3148/global.json
@@ -0,0 +1,7 @@
+{
+ "sdk": {
+ "version": "10.0.203",
+ "rollForward": "disable",
+ "allowPrerelease": false
+ }
+}
\ No newline at end of file
From 7596df7627c9084bb471f4e5d62016abe25dd695 Mon Sep 17 00:00:00 2001
From: Paul Medynski <31868385+paulmedynski@users.noreply.github.com>
Date: Fri, 24 Apr 2026 12:06:24 -0300
Subject: [PATCH 2/2] Add SNI load failure investigation probes
---
.../issue-3148/Issue3148PathProbe.csproj | 2 +
investigations/issue-3148/Program.cs | 878 +++++++++++++++++-
investigations/issue-3148/README.md | 627 ++++++++++++-
3 files changed, 1503 insertions(+), 4 deletions(-)
diff --git a/investigations/issue-3148/Issue3148PathProbe.csproj b/investigations/issue-3148/Issue3148PathProbe.csproj
index 20500a2697..748cd1faea 100644
--- a/investigations/issue-3148/Issue3148PathProbe.csproj
+++ b/investigations/issue-3148/Issue3148PathProbe.csproj
@@ -6,6 +6,8 @@
enable
latest
false
+ true
+ true
diff --git a/investigations/issue-3148/Program.cs b/investigations/issue-3148/Program.cs
index 64179daa5a..a12f3f15f0 100644
--- a/investigations/issue-3148/Program.cs
+++ b/investigations/issue-3148/Program.cs
@@ -1,4 +1,5 @@
using System.Diagnostics;
+using System.Reflection.PortableExecutable;
using System.Runtime.InteropServices;
using Microsoft.Data.SqlClient;
@@ -21,11 +22,41 @@ private static int Main(string[] args)
return 2;
}
+ if (options.StressLoadAbsent)
+ {
+ return RunStressTest(options);
+ }
+
if (options.ForceX86DotnetFirst)
{
return RunWithForcedDotnetPathOrdering(options);
}
+ if (options.ProbeSniPaths)
+ {
+ return RunProbeSniPaths();
+ }
+
+ if (options.ProbeArchMatch)
+ {
+ return RunProbeArchMatch();
+ }
+
+ if (options.ProbeNativeLoad)
+ {
+ return RunProbeNativeLoad();
+ }
+
+ if (options.ProbeExtractionRace)
+ {
+ return RunProbeExtractionRace(options);
+ }
+
+ if (options.ProbeLazyLoad)
+ {
+ return RunProbeLazyLoad(options);
+ }
+
PrintEnvironmentReport();
if (string.IsNullOrWhiteSpace(options.ConnectionString))
@@ -191,7 +222,101 @@ private static void PrintUsage()
{
Console.WriteLine();
Console.WriteLine("Usage:");
- Console.WriteLine(" Issue3148PathProbe [--connection-string ] [--force-x86-dotnet-first]");
+ Console.WriteLine(" Issue3148PathProbe [options]");
+ Console.WriteLine();
+ Console.WriteLine("Options:");
+ Console.WriteLine(" --connection-string, -c ");
+ Console.WriteLine(" SQL Server connection string for SqlClient.Open() test or stress test.");
+ Console.WriteLine(" If not provided, only environment information will be displayed.");
+ Console.WriteLine();
+ Console.WriteLine(" --force-x86-dotnet-first");
+ Console.WriteLine(" Prioritize x86 dotnet in PATH and re-launch probe in subprocess.");
+ Console.WriteLine(" Useful for testing x86 runtime behavior on x64 systems.");
+ Console.WriteLine(" Only works on Windows with both x86 and x64 dotnet installed.");
+ Console.WriteLine();
+ Console.WriteLine(" --probe-sni-paths");
+ Console.WriteLine(" Enumerate every location the runtime will probe for SNI.dll and report");
+ Console.WriteLine(" existence and file size. Covers scenario A (DLL not deployed).");
+ Console.WriteLine(" Windows-only. No connection string required.");
+ Console.WriteLine();
+ Console.WriteLine(" --probe-arch-match");
+ Console.WriteLine(" Read the PE header of each SNI.dll found on disk and compare its");
+ Console.WriteLine(" machine type to the current process architecture.");
+ Console.WriteLine(" Reports MATCH or MISMATCH for each copy. Covers scenario B (arch mismatch).");
+ Console.WriteLine(" Windows-only. No connection string required.");
+ Console.WriteLine();
+ Console.WriteLine(" --probe-native-load");
+ Console.WriteLine(" Call NativeLibrary.TryLoad() against each SNI.dll found on disk.");
+ Console.WriteLine(" A file that exists but fails to load indicates a missing VC++ dependency.");
+ Console.WriteLine(" Reports the Win32 error code on failure. Covers scenario C (missing deps).");
+ Console.WriteLine(" Windows-only. No connection string required.");
+ Console.WriteLine();
+ Console.WriteLine(" --probe-extraction-race");
+ Console.WriteLine(" Delete extracted SNI.dll copies under %TEMP% then call SqlConnection.Open().");
+ Console.WriteLine(" Reproduces scenario D (single-file extraction race / temp cleanup).");
+ Console.WriteLine(" Only meaningful when run from a single-file published exe.");
+ Console.WriteLine(" Windows-only. Requires --connection-string.");
+ Console.WriteLine();
+ Console.WriteLine(" --probe-lazy-load");
+ Console.WriteLine(" Hold the process idle, then perform the first SqlConnection.Open().");
+ Console.WriteLine(" Simulates long-running services where SNI is first loaded much later.");
+ Console.WriteLine(" Windows-only. Requires --connection-string.");
+ Console.WriteLine();
+ Console.WriteLine(" --lazy-load-delay ");
+ Console.WriteLine(" Delay before first SqlConnection.Open() in --probe-lazy-load mode.");
+ Console.WriteLine(" Default: 60");
+ Console.WriteLine();
+ Console.WriteLine(" --lazy-load-disturb-sni");
+ Console.WriteLine(" Disturb SNI load environment before first Open() by renaming every");
+ Console.WriteLine(" discovered Microsoft.Data.SqlClient.SNI.dll file, then restoring after");
+ Console.WriteLine(" the first Open() attempt completes.");
+ Console.WriteLine(" Only applies when --probe-lazy-load is used.");
+ Console.WriteLine();
+ Console.WriteLine(" --stress-load-absent");
+ Console.WriteLine(" Load-time absent probe (child-process mode).");
+ Console.WriteLine(" Proves that error 0x8007007E is elicited when SNI is absent at LoadLibrary time.");
+ Console.WriteLine(" Spawns fresh child processes so SNI is loaded from scratch each time.");
+ Console.WriteLine(" Parent renames the SNI DLL before each child starts to create a genuine absent-at-load window.");
+ Console.WriteLine(" Requires --connection-string and only supported on Windows.");
+ Console.WriteLine();
+ Console.WriteLine(" --stress-duration ");
+ Console.WriteLine(" Duration of stress test in seconds. Default: 30");
+ Console.WriteLine(" Only applies when --stress-load-absent is used.");
+ Console.WriteLine();
+ Console.WriteLine(" -h, --help, /?");
+ Console.WriteLine(" Display this help message.");
+ Console.WriteLine();
+ Console.WriteLine("Examples:");
+ Console.WriteLine(" # Display environment information only");
+ Console.WriteLine(" Issue3148PathProbe");
+ Console.WriteLine();
+ Console.WriteLine(" # Test SQL connection");
+ Console.WriteLine(" Issue3148PathProbe --connection-string \"Server=localhost;User ID=sa;Password=xxx\"");
+ Console.WriteLine();
+ Console.WriteLine(" # Enumerate SNI search paths");
+ Console.WriteLine(" Issue3148PathProbe --probe-sni-paths");
+ Console.WriteLine();
+ Console.WriteLine(" # Check SNI architecture matches process");
+ Console.WriteLine(" Issue3148PathProbe --probe-arch-match");
+ Console.WriteLine();
+ Console.WriteLine(" # Check VC++ dependencies load correctly");
+ Console.WriteLine(" Issue3148PathProbe --probe-native-load");
+ Console.WriteLine();
+ Console.WriteLine(" # Single-file extraction race (run from published exe)");
+ Console.WriteLine(" Issue3148PathProbe --probe-extraction-race --connection-string \"...\"");
+ Console.WriteLine();
+ Console.WriteLine(" # Lazy first-load repro (delay then first Open)");
+ Console.WriteLine(" Issue3148PathProbe --probe-lazy-load --lazy-load-delay 600 --connection-string \"...\"");
+ Console.WriteLine();
+ Console.WriteLine(" # Lazy first-load repro with intentional disturbance");
+ Console.WriteLine(" Issue3148PathProbe --probe-lazy-load --lazy-load-delay 30 --lazy-load-disturb-sni --connection-string \"...\"");
+ Console.WriteLine();
+ Console.WriteLine(" # Load-time absent probe for 60 seconds");
+ Console.WriteLine(" Issue3148PathProbe --stress-load-absent --connection-string \"...\" --stress-duration 60");
+ Console.WriteLine();
+ Console.WriteLine(" # Force x86 dotnet and test");
+ Console.WriteLine(" Issue3148PathProbe --force-x86-dotnet-first --connection-string \"...\"");
+ Console.WriteLine();
}
private static int RunWithForcedDotnetPathOrdering(Options options)
@@ -242,6 +367,375 @@ private static int RunWithForcedDotnetPathOrdering(Options options)
return child.ExitCode;
}
+ // -------------------------------------------------------------------------
+ // Probe A: enumerate every candidate SNI path and report existence + size
+ // -------------------------------------------------------------------------
+ private static int RunProbeSniPaths()
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--probe-sni-paths is only supported on Windows.");
+ return 2;
+ }
+
+ Console.WriteLine("Probe: SNI path survey");
+ Console.WriteLine(new string('=', 22));
+ Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}");
+ Console.WriteLine();
+
+ var paths = FindSniDllPaths().ToList();
+
+ if (paths.Count == 0)
+ {
+ Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found in any search location.");
+ Console.WriteLine();
+ Console.WriteLine("Search roots checked:");
+ foreach (string root in GetSniSearchRoots())
+ {
+ Console.WriteLine($" {root} (exists={Directory.Exists(root)})");
+ }
+ return 1;
+ }
+
+ foreach (string dll in paths)
+ {
+ long size = new FileInfo(dll).Length;
+ Console.WriteLine($"[FOUND] {dll}");
+ Console.WriteLine($" size={size} bytes");
+ }
+
+ Console.WriteLine();
+ return 0;
+ }
+
+ // -------------------------------------------------------------------------
+ // Probe B: PE header inspection — architecture match
+ // -------------------------------------------------------------------------
+ private static int RunProbeArchMatch()
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--probe-arch-match is only supported on Windows.");
+ return 2;
+ }
+
+ Console.WriteLine("Probe: SNI architecture match");
+ Console.WriteLine(new string('=', 29));
+
+ Architecture processArch = RuntimeInformation.ProcessArchitecture;
+ Console.WriteLine($"Process architecture : {processArch}");
+ Console.WriteLine();
+
+ var paths = FindSniDllPaths().ToList();
+ if (paths.Count == 0)
+ {
+ Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found — cannot check architecture.");
+ return 1;
+ }
+
+ int exitCode = 0;
+ foreach (string dll in paths)
+ {
+ Machine? machine = ReadPeMachine(dll);
+ if (machine is null)
+ {
+ Console.WriteLine($"[ERROR ] {dll}");
+ Console.WriteLine($" Could not read PE header.");
+ exitCode = 1;
+ continue;
+ }
+
+ string dllArch = machine.Value switch
+ {
+ Machine.I386 => "X86",
+ Machine.Amd64 => "X64",
+ Machine.Arm => "Arm",
+ Machine.Arm64 => "Arm64",
+ _ => $"unknown (0x{(ushort)machine.Value:X4})"
+ };
+
+ bool match = processArch switch
+ {
+ Architecture.X86 => machine.Value == Machine.I386,
+ Architecture.X64 => machine.Value == Machine.Amd64,
+ Architecture.Arm => machine.Value == Machine.Arm,
+ Architecture.Arm64 => machine.Value == Machine.Arm64,
+ _ => false
+ };
+
+ string verdict = match ? "[MATCH ]" : "[MISMATCH ]";
+ Console.WriteLine($"{verdict} {dll}");
+ Console.WriteLine($" DLL arch={dllArch} process arch={processArch}");
+
+ if (!match)
+ {
+ exitCode = 1;
+ }
+ }
+
+ Console.WriteLine();
+ return exitCode;
+ }
+
+ private static Machine? ReadPeMachine(string dllPath)
+ {
+ try
+ {
+ using FileStream fs = File.OpenRead(dllPath);
+ using PEReader pe = new(fs);
+ return pe.PEHeaders.CoffHeader.Machine;
+ }
+ catch
+ {
+ return null;
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Probe C: NativeLibrary.TryLoad — surfaces missing VC++ dependencies
+ // -------------------------------------------------------------------------
+ private static int RunProbeNativeLoad()
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--probe-native-load is only supported on Windows.");
+ return 2;
+ }
+
+ Console.WriteLine("Probe: NativeLibrary.TryLoad");
+ Console.WriteLine(new string('=', 27));
+ Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}");
+ Console.WriteLine();
+
+ var paths = FindSniDllPaths().ToList();
+ if (paths.Count == 0)
+ {
+ Console.WriteLine("[ABSENT] No Microsoft.Data.SqlClient.SNI.dll found — nothing to load.");
+ return 1;
+ }
+
+ int exitCode = 0;
+ foreach (string dll in paths)
+ {
+ bool loaded = NativeLibrary.TryLoad(dll, out IntPtr handle);
+ if (loaded)
+ {
+ Console.WriteLine($"[OK ] {dll}");
+ NativeLibrary.Free(handle);
+ }
+ else
+ {
+ // NativeLibrary.TryLoad routes through managed exception handling, so the
+ // raw Win32 error is not accessible via GetLastWin32Error(). Diagnose the
+ // failure using our own PE header inspection instead.
+ Machine? machine = ReadPeMachine(dll);
+ Architecture processArch = RuntimeInformation.ProcessArchitecture;
+ bool archMismatch = machine is not null && !(processArch switch
+ {
+ Architecture.X86 => machine.Value == Machine.I386,
+ Architecture.X64 => machine.Value == Machine.Amd64,
+ Architecture.Arm => machine.Value == Machine.Arm,
+ Architecture.Arm64 => machine.Value == Machine.Arm64,
+ _ => false
+ });
+
+ Console.WriteLine($"[FAIL ] {dll}");
+ if (archMismatch)
+ {
+ string dllArch = machine!.Value switch
+ {
+ Machine.I386 => "X86",
+ Machine.Amd64 => "X64",
+ Machine.Arm => "Arm",
+ Machine.Arm64 => "Arm64",
+ _ => $"0x{(ushort)machine.Value:X4}"
+ };
+ Console.WriteLine($" Architecture mismatch: DLL={dllArch}, process={processArch}.");
+ Console.WriteLine($" Windows will report ERROR_BAD_EXE_FORMAT (193 / 0xC1) to the CLR.");
+ Console.WriteLine($" Use --probe-arch-match for a full arch survey.");
+ }
+ else
+ {
+ Console.WriteLine($" DLL exists and arch looks correct but load failed.");
+ Console.WriteLine($" Likely cause: a VC++ or other import-table dependency is missing.");
+ Console.WriteLine($" Windows returns ERROR_MOD_NOT_FOUND (126 / 0x7E) to the CLR.");
+ Console.WriteLine($" Install the Visual C++ redistributable matching the SNI DLL version.");
+ }
+ exitCode = 1;
+ }
+ }
+
+ Console.WriteLine();
+ return exitCode;
+ }
+
+ // -------------------------------------------------------------------------
+ // Probe D: single-file extraction race — delete extracted SNI then Open()
+ // -------------------------------------------------------------------------
+ private static int RunProbeExtractionRace(Options options)
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--probe-extraction-race is only supported on Windows.");
+ return 2;
+ }
+
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ Console.Error.WriteLine("--probe-extraction-race requires --connection-string.");
+ return 2;
+ }
+
+ Console.WriteLine("Probe: single-file extraction race");
+ Console.WriteLine(new string('=', 34));
+ Console.WriteLine($"Process architecture: {RuntimeInformation.ProcessArchitecture}");
+ Console.WriteLine();
+
+ // Find SNI DLLs under %TEMP% only — those are single-file extraction candidates.
+ string tempRoot = Path.GetTempPath();
+ List extracted = [];
+ try
+ {
+ extracted = Directory.GetFiles(tempRoot, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories).ToList();
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine($"Could not enumerate temp directory: {ex.Message}");
+ }
+
+ if (extracted.Count == 0)
+ {
+ Console.WriteLine("No extracted SNI DLL found under %TEMP%.");
+ Console.WriteLine("This probe is only meaningful for single-file published apps.");
+ Console.WriteLine("Publish with -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true,");
+ Console.WriteLine("run the published exe once so the runtime extracts SNI to %TEMP%,");
+ Console.WriteLine("then re-run this probe to delete the extracted copy before SqlConnection.Open().");
+ return 2;
+ }
+
+ // Delete the extracted copies. This process has never loaded SNI, so no sharing violation.
+ List deleted = [];
+ foreach (string dll in extracted)
+ {
+ try
+ {
+ File.Delete(dll);
+ deleted.Add(dll);
+ Console.WriteLine($"[DELETED] {dll}");
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine($"[SKIP ] {dll} — {ex.Message}");
+ }
+ }
+
+ if (deleted.Count == 0)
+ {
+ Console.WriteLine("Could not delete any extracted SNI DLL (files may be in use by another process).");
+ return 2;
+ }
+
+ Console.WriteLine();
+ Console.WriteLine($"Deleted {deleted.Count} extracted SNI DLL(s). Attempting SqlConnection.Open()...");
+ Console.WriteLine("If this is a framework-dependent app the runtime will find SNI via the packs cache and succeed.");
+ Console.WriteLine("Run this probe from a single-file published exe to reproduce the actual race.");
+ Console.WriteLine();
+
+ try
+ {
+ using SqlConnection connection = new(options.ConnectionString);
+ connection.Open();
+ Console.WriteLine("SqlConnection.Open() succeeded (SNI found via fallback search path).");
+ return 0;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("SqlConnection.Open() failed — SNI absent at LoadLibrary time.");
+ PrintExceptionTree(ex);
+ return 1;
+ }
+ }
+
+ // -------------------------------------------------------------------------
+ // Probe E: delayed first load — long-running service simulation
+ // -------------------------------------------------------------------------
+ private static int RunProbeLazyLoad(Options options)
+ {
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--probe-lazy-load is only supported on Windows.");
+ return 2;
+ }
+
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ Console.Error.WriteLine("--probe-lazy-load requires --connection-string.");
+ return 2;
+ }
+
+ int delaySeconds = options.LazyLoadDelaySeconds;
+ if (delaySeconds < 0)
+ {
+ Console.Error.WriteLine("--lazy-load-delay must be >= 0.");
+ return 2;
+ }
+
+ PrintEnvironmentReport();
+ Console.WriteLine();
+ Console.WriteLine("Probe: delayed first SNI load");
+ Console.WriteLine(new string('=', 30));
+ Console.WriteLine($"PID: {Environment.ProcessId}");
+ Console.WriteLine($"Delay before first SqlConnection.Open(): {delaySeconds} seconds");
+ Console.WriteLine("During this delay, you can simulate environment drift (temp cleanup, deploy changes, AV actions).");
+ Console.WriteLine();
+
+ DateTime start = DateTime.UtcNow;
+ DateTime end = start.AddSeconds(delaySeconds);
+
+ while (DateTime.UtcNow < end)
+ {
+ int remaining = (int)Math.Ceiling((end - DateTime.UtcNow).TotalSeconds);
+ Console.WriteLine($"Waiting... {remaining}s remaining");
+ System.Threading.Thread.Sleep(1000);
+ }
+
+ List<(string Original, string Temp)> disturbed = [];
+ if (options.LazyLoadDisturbSni)
+ {
+ Console.WriteLine();
+ Console.WriteLine("Disturbance enabled: renaming discovered SNI DLL files before first Open()...");
+ disturbed = RenameAllSniDllsForLazyLoad(".lazy_disturb");
+ if (disturbed.Count == 0)
+ {
+ Console.WriteLine("No SNI files were renamed. Disturbance may have had no effect.");
+ }
+ else
+ {
+ Console.WriteLine($"Renamed {disturbed.Count} SNI file(s). First Open() will run while they are absent.");
+ }
+ }
+
+ Console.WriteLine();
+ Console.WriteLine("Delay complete. Attempting first SqlConnection.Open() now...");
+
+ int exitCode;
+ try
+ {
+ exitCode = RunConnectionTest(options.ConnectionString);
+ }
+ finally
+ {
+ if (disturbed.Count > 0)
+ {
+ RestoreSniDlls(disturbed);
+ Console.WriteLine();
+ Console.WriteLine($"Restored {disturbed.Count} disturbed SNI file(s).");
+ }
+ }
+
+ return exitCode;
+ }
+
private static int RunConnectionTest(string? connectionString)
{
if (string.IsNullOrWhiteSpace(connectionString))
@@ -316,15 +810,331 @@ private static string BuildChildArguments(Options options)
private static string QuoteArgument(string value)
=> $"\"{value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal)}\"";
+ private static int RunStressTest(Options options)
+ {
+ if (string.IsNullOrWhiteSpace(options.ConnectionString))
+ {
+ Console.Error.WriteLine("--stress-load-absent requires --connection-string.");
+ return 2;
+ }
+
+ if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+ {
+ Console.Error.WriteLine("--stress-load-absent is only supported on Windows.");
+ return 2;
+ }
+
+ // Find the assembly to re-launch as children.
+ // When running as `dotnet foo.dll`, Environment.ProcessPath is the dotnet host, not the
+ // assembly. Use Assembly.Location which always points to the .dll (empty in single-file).
+ string assemblyLocation = typeof(Program).Assembly.Location;
+ bool isDll = !string.IsNullOrEmpty(assemblyLocation)
+ && assemblyLocation.EndsWith(".dll", StringComparison.OrdinalIgnoreCase);
+
+ string childTarget;
+ string launcher;
+
+ if (isDll)
+ {
+ childTarget = assemblyLocation;
+ launcher = ResolveCommandFromPath(GetDotnetCommandName())
+ ?? throw new InvalidOperationException("Cannot resolve dotnet executable from PATH.");
+ }
+ else
+ {
+ // Self-contained or single-file exe — the process itself is the launcher.
+ childTarget = Environment.ProcessPath
+ ?? throw new InvalidOperationException("Cannot determine the current process path.");
+ launcher = childTarget;
+ }
+
+ PrintEnvironmentReport();
+ Console.WriteLine();
+ Console.WriteLine("Starting load-time absent probe (child-process mode)...");
+ Console.WriteLine($"Duration: {options.StressDurationSeconds} seconds");
+ Console.WriteLine("Each iteration spawns a fresh child process so SNI loads from scratch.");
+ Console.WriteLine("Parent renames SNI DLLs before each child starts; the child sees SNI absent at LoadLibrary time.");
+ Console.WriteLine();
+
+ DateTime endTime = DateTime.UtcNow.AddSeconds(options.StressDurationSeconds);
+ int successCount = 0;
+ int failureCount = 0;
+ int renameWindows = 0;
+ int iteration = 0;
+
+ while (DateTime.UtcNow < endTime)
+ {
+ iteration++;
+
+ // Rename SNI DLLs so they are temporarily unavailable when the child process starts.
+ // The parent can do this because it has never loaded SNI itself.
+ List<(string Original, string Temp)> renamed = RenameAllSniDlls(".stress_rename");
+ if (renamed.Count > 0)
+ {
+ renameWindows++;
+ }
+
+ try
+ {
+ ProcessStartInfo startInfo = new()
+ {
+ UseShellExecute = false,
+ // Redirect child output so it doesn't flood the parent console.
+ RedirectStandardOutput = true,
+ RedirectStandardError = true
+ };
+
+ if (isDll)
+ {
+ startInfo.FileName = launcher;
+ startInfo.ArgumentList.Add(childTarget);
+ }
+ else
+ {
+ startInfo.FileName = launcher;
+ }
+
+ startInfo.ArgumentList.Add("--connection-string");
+ startInfo.ArgumentList.Add(options.ConnectionString);
+
+ using Process child = Process.Start(startInfo)
+ ?? throw new InvalidOperationException("Failed to start child process.");
+
+ // Drain stdout/stderr async to prevent deadlock if the pipe buffer fills.
+ var stdoutDrain = child.StandardOutput.ReadToEndAsync();
+ var stderrDrain = child.StandardError.ReadToEndAsync();
+
+ // Hold the rename window for up to 2 seconds — long enough for the child to
+ // start the CLR and call LoadLibrary for SNI — then restore so that if SNI
+ // is not yet loaded, the child can still pick it up from disk.
+ System.Threading.Thread.Sleep(2000);
+ RestoreSniDlls(renamed);
+ renamed = [];
+
+ child.WaitForExit();
+ stdoutDrain.GetAwaiter().GetResult();
+ stderrDrain.GetAwaiter().GetResult();
+
+ if (child.ExitCode == 0)
+ {
+ successCount++;
+ Console.WriteLine($"[OK] iteration={iteration}");
+ }
+ else
+ {
+ failureCount++;
+ Console.WriteLine($"[FAIL] iteration={iteration} exitCode={child.ExitCode}");
+ }
+ }
+ finally
+ {
+ // Always restore — even if Process.Start or WaitForExit throws.
+ RestoreSniDlls(renamed);
+ }
+ }
+
+ Console.WriteLine();
+ Console.WriteLine("Stress test complete.");
+ Console.WriteLine($"Child process launches: {iteration}");
+ Console.WriteLine($" Successful: {successCount}");
+ Console.WriteLine($" Failed: {failureCount}");
+ Console.WriteLine($"Iterations with rename window: {renameWindows} of {iteration}");
+
+ return failureCount > 0 ? 1 : 0;
+ }
+
+ // Returns every SNI DLL found in on-disk search locations (packs cache and temp extraction
+ // directories), renamed to a temporary name. The parent process — which has never loaded
+ // SqlClient — can rename these without hitting a sharing violation.
+ private static List<(string Original, string Temp)> RenameAllSniDlls(string suffix)
+ => RenameSniDlls(FindSniDllPaths(), suffix);
+
+ private static List<(string Original, string Temp)> RenameAllSniDllsForLazyLoad(string suffix)
+ => RenameSniDlls(FindSniDllPathsForLazyLoadDisturbance(), suffix);
+
+ private static List<(string Original, string Temp)> RenameSniDlls(IEnumerable dllPaths, string suffix)
+ {
+ List<(string, string)> renamed = [];
+
+ foreach (string dll in dllPaths)
+ {
+ string temp = dll + suffix;
+ try
+ {
+ File.Move(dll, temp);
+ renamed.Add((dll, temp));
+ }
+ catch (Exception)
+ {
+ // File in use (loaded by another process), inaccessible, or permission-denied — skip.
+ }
+ }
+
+ return renamed;
+ }
+
+ private static IEnumerable FindSniDllPathsForLazyLoadDisturbance()
+ {
+ HashSet discovered = new(StringComparer.OrdinalIgnoreCase);
+
+ // Include the baseline roots used by other probes.
+ foreach (string dll in FindSniDllPaths())
+ {
+ discovered.Add(dll);
+ }
+
+ // Include direct app-local candidates where framework-dependent outputs place native assets.
+ string[] directCandidates =
+ [
+ Path.Combine(AppContext.BaseDirectory, "Microsoft.Data.SqlClient.SNI.dll"),
+ Path.Combine(AppContext.BaseDirectory, "runtimes", "win-x64", "native", "Microsoft.Data.SqlClient.SNI.dll"),
+ Path.Combine(AppContext.BaseDirectory, "runtimes", "win-x86", "native", "Microsoft.Data.SqlClient.SNI.dll"),
+ Path.Combine(AppContext.BaseDirectory, "runtimes", "win-arm64", "native", "Microsoft.Data.SqlClient.SNI.dll")
+ ];
+
+ foreach (string candidate in directCandidates)
+ {
+ if (File.Exists(candidate))
+ {
+ discovered.Add(candidate);
+ }
+ }
+
+ // Include runtime-directed native search paths and common NuGet package roots.
+ foreach (string root in GetLazyLoadDisturbanceRoots())
+ {
+ if (!Directory.Exists(root))
+ {
+ continue;
+ }
+
+ string[] files;
+ try
+ {
+ files = Directory.GetFiles(root, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories);
+ }
+ catch (Exception)
+ {
+ continue;
+ }
+
+ foreach (string file in files)
+ {
+ discovered.Add(file);
+ }
+ }
+
+ return discovered;
+ }
+
+ private static IEnumerable GetLazyLoadDisturbanceRoots()
+ {
+ HashSet roots = new(StringComparer.OrdinalIgnoreCase)
+ {
+ AppContext.BaseDirectory,
+ Path.Combine(AppContext.BaseDirectory, "runtimes")
+ };
+
+ string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+ if (!string.IsNullOrWhiteSpace(userProfile))
+ {
+ roots.Add(Path.Combine(userProfile, ".nuget", "packages", "microsoft.data.sqlclient.sni.runtime"));
+ roots.Add(Path.Combine(userProfile, ".nuget", "packages", "microsoft.data.sqlclient.sni"));
+ }
+
+ string? nativeSearchDirs = Environment.GetEnvironmentVariable("NATIVE_DLL_SEARCH_DIRECTORIES");
+ if (!string.IsNullOrWhiteSpace(nativeSearchDirs))
+ {
+ string[] entries = nativeSearchDirs
+ .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+
+ foreach (string entry in entries)
+ {
+ roots.Add(entry);
+ }
+ }
+
+ return roots;
+ }
+
+ private static void RestoreSniDlls(List<(string Original, string Temp)> renamed)
+ {
+ foreach ((string original, string temp) in renamed)
+ {
+ try
+ {
+ if (File.Exists(temp))
+ {
+ File.Move(temp, original, overwrite: true);
+ }
+ }
+ catch (Exception)
+ {
+ // Best-effort; log nothing so as not to pollute stress output.
+ }
+ }
+ }
+
+ private static string[] GetSniSearchRoots() =>
+ [
+ Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "dotnet", "packs"),
+ Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "dotnet", "packs"),
+ Path.GetTempPath()
+ ];
+
+ private static IEnumerable FindSniDllPaths()
+ {
+ foreach (string root in GetSniSearchRoots())
+ {
+ if (!Directory.Exists(root))
+ {
+ continue;
+ }
+
+ string[] files;
+ try
+ {
+ files = Directory.GetFiles(root, "Microsoft.Data.SqlClient.SNI.dll", SearchOption.AllDirectories);
+ }
+ catch (Exception)
+ {
+ continue;
+ }
+
+ foreach (string file in files)
+ {
+ yield return file;
+ }
+ }
+ }
+
private sealed class Options
{
public string? ConnectionString { get; private init; }
public bool ForceX86DotnetFirst { get; private init; }
+ public bool ProbeSniPaths { get; private init; }
+ public bool ProbeArchMatch { get; private init; }
+ public bool ProbeNativeLoad { get; private init; }
+ public bool ProbeExtractionRace { get; private init; }
+ public bool ProbeLazyLoad { get; private init; }
+ public int LazyLoadDelaySeconds { get; private init; } = 60;
+ public bool LazyLoadDisturbSni { get; private init; }
+ public bool StressLoadAbsent { get; private init; }
+ public int StressDurationSeconds { get; private init; } = 30;
public static Options Parse(string[] args)
{
string? connectionString = null;
bool forceX86DotnetFirst = false;
+ bool probeSniPaths = false;
+ bool probeArchMatch = false;
+ bool probeNativeLoad = false;
+ bool probeExtractionRace = false;
+ bool probeLazyLoad = false;
+ int lazyLoadDelaySeconds = 60;
+ bool lazyLoadDisturbSni = false;
+ bool stressTestSni = false;
+ int stressDurationSeconds = 30;
for (int i = 0; i < args.Length; i++)
{
@@ -344,6 +1154,59 @@ public static Options Parse(string[] args)
forceX86DotnetFirst = true;
break;
+ case "--probe-sni-paths":
+ probeSniPaths = true;
+ break;
+
+ case "--probe-arch-match":
+ probeArchMatch = true;
+ break;
+
+ case "--probe-native-load":
+ probeNativeLoad = true;
+ break;
+
+ case "--probe-extraction-race":
+ probeExtractionRace = true;
+ break;
+
+ case "--probe-lazy-load":
+ probeLazyLoad = true;
+ break;
+
+ case "--lazy-load-delay":
+ if (i + 1 >= args.Length || !int.TryParse(args[i + 1], out int lazyDelay))
+ {
+ throw new ArgumentException("Invalid or missing value for --lazy-load-delay.");
+ }
+
+ if (lazyDelay < 0)
+ {
+ throw new ArgumentException("--lazy-load-delay must be >= 0.");
+ }
+
+ lazyLoadDelaySeconds = lazyDelay;
+ i++;
+ break;
+
+ case "--lazy-load-disturb-sni":
+ lazyLoadDisturbSni = true;
+ break;
+
+ case "--stress-load-absent":
+ stressTestSni = true;
+ break;
+
+ case "--stress-duration":
+ if (i + 1 >= args.Length || !int.TryParse(args[i + 1], out int duration))
+ {
+ throw new ArgumentException("Invalid or missing value for --stress-duration.");
+ }
+
+ stressDurationSeconds = duration;
+ i++;
+ break;
+
case "--help":
case "-h":
case "/?":
@@ -357,8 +1220,17 @@ public static Options Parse(string[] args)
return new Options
{
ConnectionString = connectionString,
- ForceX86DotnetFirst = forceX86DotnetFirst
+ ForceX86DotnetFirst = forceX86DotnetFirst,
+ ProbeSniPaths = probeSniPaths,
+ ProbeArchMatch = probeArchMatch,
+ ProbeNativeLoad = probeNativeLoad,
+ ProbeExtractionRace = probeExtractionRace,
+ ProbeLazyLoad = probeLazyLoad,
+ LazyLoadDelaySeconds = lazyLoadDelaySeconds,
+ LazyLoadDisturbSni = lazyLoadDisturbSni,
+ StressLoadAbsent = stressTestSni,
+ StressDurationSeconds = stressDurationSeconds
};
}
}
-}
\ No newline at end of file
+}
diff --git a/investigations/issue-3148/README.md b/investigations/issue-3148/README.md
index d40ca97262..c9c31d413b 100644
--- a/investigations/issue-3148/README.md
+++ b/investigations/issue-3148/README.md
@@ -43,6 +43,9 @@ This mode must be run as a framework-dependent app through `dotnet`, for example
`dotnet run` or `dotnet Issue3148PathProbe.dll`. It does not validate host selection for
published executables directly.
+If both `Program Files (x86)\\dotnet` and `Program Files\\dotnet` are not present in
+`PATH`, the probe prints a diagnostic message and exits with code `2`.
+
```bash
dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \
--force-x86-dotnet-first
@@ -56,6 +59,303 @@ dotnet run --project investigations/issue-3148/Issue3148PathProbe.csproj -- \
--connection-string "Server=...;Database=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True"
```
+## Test with x86 runtime only (no x86 SDK required)
+
+For validating runtime behavior, run the built DLL directly with x86 `dotnet`.
+This avoids SDK resolution and therefore avoids `global.json` SDK pinning.
+
+### 1) Verify x86 runtime installation
+
+```powershell
+$x86Dotnet = 'C:\Program Files (x86)\dotnet\dotnet.exe'
+if (Test-Path $x86Dotnet) {
+ & $x86Dotnet --list-runtimes
+}
+```
+
+Look for `Microsoft.NETCore.App 8.0.26` in the output.
+
+### 2) Ensure the probe DLL exists
+
+```powershell
+$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll'
+if (-not (Test-Path $dll)) {
+ dotnet build investigations/issue-3148/Issue3148PathProbe.csproj -v minimal
+}
+```
+
+### 3) Run with x86 host and x86-first PATH ordering
+
+```powershell
+$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll'
+$env:PATH = 'C:\Program Files (x86)\dotnet;C:\Program Files\dotnet;' + $env:PATH
+& 'C:\Program Files (x86)\dotnet\dotnet.exe' $dll
+Write-Output "EXIT:$LASTEXITCODE"
+```
+
+Expected signals in output:
+
+- `Process architecture: X86`
+- `Framework: .NET 8.0.26`
+- `Current process path: C:\Program Files (x86)\dotnet\dotnet.exe`
+- `PATH ordering assessment: warning: x86 dotnet appears before x64 in PATH ...`
+- `EXIT:0`
+
+## Scenario probes
+
+The four probes below each target one of the root-cause scenarios (A–D) identified
+in the runtime analysis. They require no SQL Server connection and run in seconds.
+
+### Probe A — SNI path survey (`--probe-sni-paths`)
+
+Enumerates every location the runtime will probe for `Microsoft.Data.SqlClient.SNI.dll`
+and reports existence and file size. Covers scenario A (DLL not deployed).
+
+```powershell
+$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll'
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-sni-paths
+```
+
+Expected output on a healthy machine:
+
+```
+Probe: SNI path survey
+======================
+Process architecture: X64
+
+[FOUND] C:\Users\...\AppData\Local\Temp\.net\Issue3148PathProbe\\Microsoft.Data.SqlClient.SNI.dll
+ size=566832 bytes
+```
+
+Exit code 1 with `[ABSENT]` means no SNI DLL was found anywhere — a deployment
+problem.
+
+### Probe B — Architecture match (`--probe-arch-match`)
+
+Reads the PE `IMAGE_FILE_MACHINE` field of every SNI DLL found on disk and compares
+it to `RuntimeInformation.ProcessArchitecture`. Covers scenario B (arch mismatch).
+
+```powershell
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-arch-match
+```
+
+Expected output:
+
+```
+Probe: SNI architecture match
+=============================
+Process architecture : X64
+
+[MATCH ] ...\Microsoft.Data.SqlClient.SNI.dll
+ DLL arch=X64 process arch=X64
+```
+
+A `[MISMATCH]` line means the DLL on disk is the wrong architecture for this
+process. The .NET runtime will report `ERROR_BAD_EXE_FORMAT` (193), not
+`ERROR_MOD_NOT_FOUND` (126). If an arch mismatch appears *together* with no
+matching-arch copy at all, the runtime will instead report `0x8007007E`.
+
+### Probe C — Native load (`--probe-native-load`)
+
+Calls `NativeLibrary.TryLoad()` against each SNI DLL found on disk. If the file
+exists and the arch matches but load still fails, a VC++ or other import-table
+dependency is missing. Covers scenario C (missing VC++ runtime).
+
+```powershell
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-native-load
+```
+
+Expected output:
+
+```
+Probe: NativeLibrary.TryLoad
+===========================
+Process architecture: X64
+
+[OK ] ...\Microsoft.Data.SqlClient.SNI.dll
+```
+
+A `[FAIL]` line with "Architecture mismatch" confirms scenario B. A `[FAIL]` line
+with "VC++ or other import-table dependency is missing" points to scenario C —
+install the Visual C++ redistributable that matches the SNI DLL version.
+
+### Probe D — Single-file extraction race (`--probe-extraction-race`)
+
+Deletes extracted SNI DLL copies under `%TEMP%` then calls `SqlConnection.Open()`.
+Simulates what happens when a temp-cleanup tool removes the extracted copy between
+process startup and the first database connection. Covers scenario D (single-file
+extraction race).
+
+This probe is only meaningful when run from a **single-file published exe** (see
+`Publish a Windows single-file probe` below). When run as a framework-dependent
+app, the runtime finds SNI via the packs cache and succeeds regardless.
+
+```powershell
+# Publish first:
+dotnet publish investigations/issue-3148/Issue3148PathProbe.csproj `
+ -c Release -r win-x64 `
+ -p:PublishSingleFile=true `
+ -p:IncludeNativeLibrariesForSelfExtract=true
+
+# Run the published exe once so SNI is extracted to %TEMP%:
+.\investigations\issue-3148\bin\Release\net8.0\win-x64\publish\Issue3148PathProbe.exe
+
+# Now run the probe — it deletes the extracted copy then calls Open():
+$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;'
+.\investigations\issue-3148\bin\Release\net8.0\win-x64\publish\Issue3148PathProbe.exe `
+ --probe-extraction-race --connection-string $cs
+```
+
+Expected output when the race is reproduced (no fallback copy exists):
+
+```
+[DELETED] C:\Users\...\AppData\Local\Temp\.net\Issue3148PathProbe\\Microsoft.Data.SqlClient.SNI.dll
+...
+SqlConnection.Open() failed — SNI absent at LoadLibrary time.
+[0] System.DllNotFoundException: Unable to load DLL 'Microsoft.Data.SqlClient.SNI.dll'...
+```
+
+### Probe E — Delayed first load (`--probe-lazy-load`)
+
+This mode simulates a long-running process that does not touch SqlClient until
+later. The probe waits for a configurable delay and only then performs the first
+`SqlConnection.Open()`, which is when SNI is actually loaded.
+
+Use this mode when you need to reproduce "service ran for hours, then first DB
+call failed with `0x8007007E`" behavior.
+
+```powershell
+$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll'
+$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;'
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-lazy-load --lazy-load-delay 600 --connection-string $cs
+```
+
+To intentionally disturb the SNI load environment during the delay window:
+
+```powershell
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --probe-lazy-load --lazy-load-delay 30 --lazy-load-disturb-sni --connection-string $cs
+```
+
+Options for this probe:
+
+- `--probe-lazy-load` — Enable delayed first-load probe.
+- `--lazy-load-delay ` — Delay before first `SqlConnection.Open()` (default: 60).
+- `--lazy-load-disturb-sni` — Rename discovered `Microsoft.Data.SqlClient.SNI.dll` files (app output, native search directories, and relevant NuGet package roots) just before first `Open()`, then restore them after the attempt.
+- `--connection-string` — Required.
+
+Expected output:
+
+```
+Probe: delayed first SNI load
+==============================
+PID: 12345
+Delay before first SqlConnection.Open(): 600 seconds
+Waiting... 600s remaining
+...
+Delay complete. Attempting first SqlConnection.Open() now...
+```
+
+With `--lazy-load-disturb-sni`, output will also show how many SNI DLL files were
+renamed before first `Open()`, and confirm they were restored afterward.
+
+If the environment has drifted during the wait (for example temp cleanup,
+deployment changes, or AV timing), this first-open call is where `0x8007007E`
+will surface.
+
+## Load-time absent probe (`--stress-load-absent`)
+
+This mode proves that error `0x8007007E` is elicited when SNI is absent at
+`LoadLibrary` time. Because proof #1 shows that a process which has already loaded
+SNI cannot encounter this error again, this mode tests only the **load-time path**
+— it does not reproduce the long-running service scenario.
+
+### Why in-process connection loops cannot trigger the issue
+
+Once `Microsoft.Data.SqlClient.SNI.dll` is loaded via `LoadLibrary`, Windows holds
+a lock on the file for the **lifetime of the process**. All subsequent
+`SqlConnection.Open()` calls reuse the already-loaded native library without
+touching the disk. An in-process loop therefore exercises the *post-load* path,
+which is bulletproof by design. Rename attempts against an already-loaded DLL
+always fail with a sharing-violation error.
+
+### How the child-process approach works
+
+The stress test spawns a **fresh child process** for each attempt. Each child goes
+through the full `LoadLibrary` path from scratch. The parent process — which never
+loads `SqlClient` or `SNI` itself — can rename the SNI DLL freely before the child
+starts, creating a genuine load-time interference window.
+
+| Aspect | In-process loop (old) | Child-process spawn (new) |
+|--------|----------------------|--------------------------|
+| Connection attempts | Same process (SNI already loaded) | Fresh child process each time (SNI not yet loaded) |
+| DLL rename window | After SNI loaded → always blocked (file in use) | Before child starts → succeeds every iteration |
+| What is tested | `Open()`/`Close()` throughput | Actual `LoadLibrary` call path |
+| Can trigger issue #3148? | No | Yes — if SNI is absent or wrong-arch when child calls `LoadLibrary` |
+
+The parent holds the rename for 2 seconds (long enough for the child CLR to start
+and reach `LoadLibrary`), then restores the DLL so the child can still pick it up
+if it hasn't loaded yet. This creates a race window matching the original failure
+scenario.
+
+### Run the stress test
+
+```powershell
+$dll = Join-Path $PWD 'investigations\issue-3148\bin\Debug\net8.0\Issue3148PathProbe.dll'
+$cs = 'Server=...;User ID=...;Password=...;Encrypt=True;TrustServerCertificate=True;'
+& 'C:\Program Files\dotnet\dotnet.exe' $dll --stress-load-absent --connection-string $cs --stress-duration 60
+```
+
+Options:
+
+- `--stress-load-absent` — Enable load-time absent probe. Windows-only.
+- `--connection-string` — Required. SQL Server connection string passed to each child process.
+- `--stress-duration ` — How long to keep spawning children (default: 30 seconds).
+
+Expected output:
+
+```
+Starting SNI stress test (child-process mode)...
+Duration: 60 seconds
+Each iteration spawns a fresh child process so SNI loads from scratch.
+Parent briefly renames SNI DLLs before each child starts to inject load-time interference.
+
+[OK] iteration=1
+[OK] iteration=2
+...
+
+Stress test complete.
+Child process launches: 20
+ Successful: 20
+ Failed: 0
+Iterations with rename window: 20 of 20
+```
+
+Interpreting results:
+
+- `Iterations with rename window: N of N` — confirms the parent found the SNI DLL on
+ disk and renamed it before each child. If this is `0 of N`, the DLL was not found
+ in the expected locations (check the packs cache or temp directory).
+- `[OK]` — child loaded SNI and connected successfully.
+- `[FAIL] exitCode=1` — child failed. Capturing the child's output (remove
+ `RedirectStandardOutput` temporarily) will show the full exception chain.
+
+A failing environment (wrong SNI architecture, AV interference, or a single-file
+extraction race) would produce `[FAIL]` entries here. A clean lab environment with
+the correct x64 SNI installed will typically show all `[OK]`.
+
+## Troubleshooting tips
+
+- `dotnet run` requires an SDK and is affected by `global.json`.
+- `dotnet .dll` uses runtime resolution and is the preferred path for runtime-only testing.
+- If `--force-x86-dotnet-first` prints that both x86 and x64 entries are required, add both folders to `PATH` for that session before running.
+- If you need to test SDK commands under x86 host selection, install an x86 SDK version compatible with the repository `global.json`.
+- The `--stress-test-sni` mode is Windows-only and requires a valid SQL connection string.
+- If `Iterations with rename window: 0 of N`, the SNI DLL was not found in the packs cache
+ or temp directories. Run the probe once without `--stress-test-sni` to do a connection test
+ first; the NuGet package extraction will place the DLL in a temp folder that the stress
+ test can then locate.
+
+
## Publish a Windows single-file probe
This is useful if you want to mirror the original report more closely.
@@ -77,4 +377,329 @@ dotnet publish investigations/issue-3148/Issue3148PathProbe.csproj \
- If the process architecture and runtime architecture do not match the intended
deployment, that is another strong signal.
- If `SqlConnection.Open()` fails, the app prints the full exception chain so
- the loader error code can be captured.
\ No newline at end of file
+ the loader error code can be captured.
+
+---
+
+## Background: Runtime source analysis
+
+This section documents findings from reading the .NET 8 runtime source code
+(`v8.0.0` tag on `dotnet/runtime`). The evidence below informs both the design
+of the stress test and the list of root causes to investigate on affected systems.
+
+### Proof: SNI.dll is loaded exactly once per process, never reloaded
+
+#### Entry point: `NDirectImportThunk` → `NDirectLink` (dllimport.cpp)
+
+The first time a P/Invoke method is called, control passes through
+`NDirectImportThunk`, which calls `NDirectLink`. That function calls
+`NativeLibrary::LoadLibraryFromMethodDesc`, obtains the `HMODULE`, resolves the
+entry point, and then stores both permanently in the `NDirectMethodDesc`:
+
+```cpp
+// dllimport.cpp — NDirectLink()
+NATIVE_LIBRARY_HANDLE hmod = NativeLibrary::LoadLibraryFromMethodDesc(pMD);
+// ...
+LPVOID pvTarget = NDirectGetEntryPoint(pMD, hmod);
+if (pvTarget)
+{
+ pMD->SetNDirectTarget(pvTarget); // ← written into NDirectMethodDesc once
+ fSuccess = TRUE;
+}
+```
+
+`SetNDirectTarget` writes directly into `NDirectMethodDesc::m_pWriteableData->m_pNDirectTarget`.
+Every subsequent call to that P/Invoke method jumps directly to that pointer.
+`NDirectImportThunk` — and therefore `LoadLibrary` — is **never invoked again**
+for that method.
+
+#### The `UnmanagedImageCache`: add-only, process-lifetime (nativelibrary.cpp + appdomain.cpp)
+
+Inside `LoadLibraryFromMethodDesc`, the call chain reaches `LoadNativeLibrary()`:
+
+```cpp
+// nativelibrary.cpp — LoadNativeLibrary()
+hmod = pDomain->FindUnmanagedImageInCache(wszLibName); // check cache first
+if (hmod != NULL)
+ return hmod.Extract(); // ← return immediately, no LoadLibrary call
+
+hmod = LoadNativeLibraryBySearch(pMD, pErrorTracker, wszLibName);
+if (hmod != NULL)
+{
+ pDomain->AddUnmanagedImageToCache(wszLibName, hmod); // ← store permanently
+ return hmod.Extract();
+}
+```
+
+The cache is implemented in `AppDomain::AddUnmanagedImageToCache()` (appdomain.cpp).
+It allocates the key on the loader heap (which lives for the AppDomain lifetime)
+and inserts into a hash table. There is **no `RemoveUnmanagedImageFromCache`**
+anywhere in the codebase. The cache is insert-only.
+
+In .NET 8, there is a single AppDomain that lasts for the entire process lifetime.
+Therefore:
+
+- `LoadLibrary` is called for `SNI.dll` **at most once per process**.
+- After the first successful load the HMODULE is cached; all future P/Invoke
+ calls use the cached function pointer and never touch the disk.
+- `FreeLibrary` is never called for P/Invoke-loaded libraries during normal
+ process execution. The OS keeps the DLL mapped until process exit.
+
+#### Why this matters for a long-running service
+
+Because loading is **lazy** (triggered by the first `SqlConnection.Open()`, not at
+process startup), the `LoadLibrary` call can happen at any point during the service's
+lifetime — potentially hours after startup. Whether the load succeeds depends on
+filesystem state, AV scanner activity, and environment **at that moment**.
+Subsequent connections reuse the cached pointer and never touch the disk, which is
+why affected services often recover after a process restart that re-loads SNI.dll
+under favourable conditions.
+
+---
+
+### All scenarios where error `0x8007007E` can occur
+
+`0x8007007E = HRESULT_FROM_WIN32(ERROR_MOD_NOT_FOUND)` — Win32 error 126,
+"The specified module could not be found."
+
+The runtime converts this to `DllNotFoundException` inside
+`LoadLibErrorTracker::Throw()` (nativelibrary.cpp):
+
+```cpp
+HRESULT theHRESULT = GetHR();
+if (theHRESULT == HRESULT_FROM_WIN32(ERROR_BAD_EXE_FORMAT))
+ COMPlusThrow(kBadImageFormatException);
+else
+{
+ SString hrString;
+ GetHRMsg(theHRESULT, hrString);
+ COMPlusThrow(kDllNotFoundException, IDS_EE_NDIRECT_LOADLIB_WIN,
+ libraryNameOrPath.GetUnicode(), hrString);
+}
+```
+
+The error tracker assigns priorities when multiple probes are attempted:
+
+```cpp
+case ERROR_FILE_NOT_FOUND:
+case ERROR_PATH_NOT_FOUND:
+case ERROR_MOD_NOT_FOUND: // priority 10 — lowest
+case ERROR_DLL_NOT_FOUND:
+ priority = const_priorityNotFound;
+ break;
+case ERROR_ACCESS_DENIED:
+ priority = const_priorityAccessDenied; // priority 20
+default:
+ priority = const_priorityCouldNotLoad; // priority 99999 — highest
+```
+
+The tracker keeps only the highest-priority error. Seeing `0x8007007E` in the
+exception means **every** search strategy returned "not found" or lower — no probe
+hit a higher-priority error such as access denied or a corrupt image.
+
+#### Search order that must exhaust before the exception is thrown
+
+`LoadNativeLibraryBySearch()` probes these locations in order:
+
+1. Windows API sets (skipped — SNI is not an API set name)
+2. Directories in `NATIVE_DLL_SEARCH_DIRECTORIES` (set by the host)
+3. The P/Invoke assembly's own directory (the `.dll`/`.exe` folder)
+4. `LoadLibraryEx` with `dllImportSearchPathFlags` (assembly-level or default)
+
+All of these must fail for `0x8007007E` to be reported.
+
+#### Scenario A — SNI DLL not deployed
+
+The file is simply absent from all probed paths. This is the most common root
+cause when a deployment is incomplete (missing the
+`runtimes/win-x64/native/` or `runtimes/win-x86/native/` subtree from the NuGet
+layout). Results in a consistent, non-intermittent failure on every
+`SqlConnection.Open()`.
+
+#### Scenario B — Architecture mismatch: no matching-arch DLL present
+
+A 32-bit service process on a machine that has only the x64 NuGet layout (or vice
+versa). The runtime probes for a DLL whose bitness matches the process, finds
+none, and reports `0x8007007E`. If the wrong-arch DLL is found and Windows tries
+to load it into the mismatched process, `ERROR_BAD_EXE_FORMAT` (193) is returned
+instead, producing `BadImageFormatException` rather than `DllNotFoundException`.
+Getting `0x8007007E` with an arch mismatch therefore means there is no DLL at all
+for that architecture — not even a wrong-arch one.
+
+This is the scenario most consistent with issue #3148's description: a Windows
+service that runs as 32-bit (`x86`) on a machine whose deployment only includes
+the `win-x64` NuGet runtime assets.
+
+#### Scenario C — A dependency of SNI.dll is missing
+
+`Microsoft.Data.SqlClient.SNI.dll` is a native C++ DLL with its own import table.
+It depends on system DLLs and the Visual C++ runtime (`VCRUNTIME140.dll`,
+`MSVCP140.dll`, etc.). When `LoadLibrary` maps SNI.dll, the Windows loader
+recursively resolves its imports. If any **direct or transitive** dependency is
+missing, the Windows loader returns `ERROR_MOD_NOT_FOUND` for the outer call to
+`SNI.dll` — even though SNI.dll itself was found on disk. The missing dependency
+is not identified in the exception message.
+
+Common in stripped server images, minimal Docker base images, or servers that
+lack the Visual C++ redistributable.
+
+#### Scenario D — Single-file app extraction race
+
+When published with `PublishSingleFile=true` and
+`IncludeNativeLibrariesForSelfExtract=true`, native DLLs are extracted to a
+temp directory at startup (e.g.,
+`%TEMP%\.net\\\`). Extraction and `LoadLibrary` are two
+distinct operations. If a cleanup tool, AV scanner, or another process removes
+the extracted file in the window between extraction and the first
+`SqlConnection.Open()`, the file is gone when `LoadLibrary` is called →
+`ERROR_MOD_NOT_FOUND`. This is intermittent and can occur hours after startup
+if the service's first database connection is delayed.
+
+#### Scenario E — AV/EDR scan-on-access interception
+
+Some endpoint security products intercept `LoadLibrary` calls by briefly renaming
+or locking the target file during an on-access scan. From the calling process's
+perspective the file disappears between the search pass and the map pass, and the
+OS loader reports `ERROR_MOD_NOT_FOUND`. This failure is intermittent and
+timing-dependent — it occurs only when the AV scanner's scan window overlaps with
+the `LoadLibrary` call, which can happen at any point during the process lifetime
+(including long after startup).
+
+This is the scenario the `--stress-load-absent` mode's rename-window mechanism is designed to
+simulate.
+
+#### Scenario F — Directory-level ACL change after deployment
+
+If the runtime probes the assembly directory but the **directory** itself (not just
+the DLL file) loses read/traverse permission after deployment, the probe silently
+finds nothing and reports `ERROR_MOD_NOT_FOUND`. If the DLL file itself loses read
+permission, `ERROR_ACCESS_DENIED` is reported instead (and would override
+`0x8007007E` in the error tracker). The directory case therefore produces
+`0x8007007E` even though the file physically exists.
+
+#### Scenario G — SxS/manifest activation failure (rare)
+
+`Microsoft.Data.SqlClient.SNI.dll` ships with an embedded side-by-side manifest
+specifying a VC++ runtime version. If the activation context creation fails (e.g.,
+the required VC++ merge module is not installed on the system), Windows returns
+`ERROR_MOD_NOT_FOUND` for the outer `LoadLibrary` call rather than a more specific
+SxS error. This is uncommon in modern deployments but can occur on images built
+with minimal Windows component selection.
+
+#### Summary
+
+| Scenario | Error consistent? | Intermittent? | Most relevant to issue #3148? |
+|---|---|---|---|
+| **A** — SNI DLL not deployed | Consistent | No | Unlikely (service ran before) |
+| **B** — Arch mismatch, no matching-arch DLL | Consistent | No | **Yes — 32-bit service, x64-only deploy** |
+| **C** — VC++ runtime dependency missing | Consistent | No | Possible on minimal images |
+| **D** — Single-file extraction race | Intermittent | Yes | Yes — if using single-file publish |
+| **E** — AV/EDR scan-on-access | Intermittent | Yes | **Yes — matches intermittent pattern** |
+| **F** — Directory ACL change | Intermittent | Yes | Possible in hardened environments |
+| **G** — SxS/manifest version mismatch | Consistent | No | Unlikely |
+
+Scenarios B, D, and E are the primary candidates for the "works after restart,
+fails intermittently in a long-running service" pattern described in issue #3148.
+
+---
+
+### Interpreting the long-running-process symptom
+
+Issue #3148 describes a process that runs for a long time, performs database work,
+and only later reports `0x8007007E` for `Microsoft.Data.SqlClient.SNI.dll`.
+
+Given the runtime behavior above (native DLL load is cached per process and never
+reloaded), a true "loaded successfully and then failed later in the same process"
+sequence is not expected in normal operation.
+
+In practice, the symptom usually means one of the following:
+
+1. The first SNI load was delayed (lazy load), so the process had been running but
+ had not yet reached the first `SqlConnection.Open()` that triggers `LoadLibrary`.
+2. The reported failure came from another process instance, worker, or recycle event,
+ while logs were interpreted as one continuous process.
+3. A non-standard event occurred (for example process/host recycle, unusual loading
+ context transitions, or environmental interference around first load).
+
+#### What "something extraordinary happens" means
+
+In this context, "extraordinary" means an event that breaks the normal assumption
+of one stable OS process with one successful first native load that remains cached.
+
+Concrete examples:
+
+1. **Silent process replacement (most common)**
+ - Service restart, app pool recycle, container restart, watchdog restart, crash-restart.
+ - Operationally appears as one continuous service, but it is a *new process* doing
+ a fresh first `LoadLibrary`.
+ - Evidence to collect: PID changes, startup timestamps, service control events,
+ container instance IDs.
+
+2. **"Successful earlier work" did not include SNI load yet**
+ - Service was active for hours, but first SqlClient path was delayed.
+ - Failure appears "late" only because first `SqlConnection.Open()` happened late.
+ - Evidence to collect: application timeline showing first DB call aligns with failure.
+
+3. **Failure occurred in a different worker process**
+ - Parent process appears healthy; child worker/sidecar process is the one failing
+ its first load.
+ - Evidence to collect: executable path + PID in logs for each failure record.
+
+4. **Host environment changed before a new first-load event**
+ - During recycle/deploy window, path/deployment/temp/AV state changed.
+ - New process then fails first load with `0x8007007E`.
+ - Evidence to collect: deployment history, temp cleanup tasks, AV quarantine logs,
+ file inventory before/after restart.
+
+5. **In-process native tampering (rare)**
+ - Injected tools/hooks alter loader behavior or call `FreeLibrary` unexpectedly.
+ - This can violate normal runtime invariants.
+ - Evidence to collect: unexpected native modules, endpoint instrumentation,
+ debugger/injector presence.
+
+6. **Severe memory corruption (very rare)**
+ - Unsafe/native corruption damages runtime or loader state.
+ - Not specific to SNI; usually accompanied by broader instability.
+ - Evidence to collect: access violations, Watson dumps, random unrelated faults.
+
+7. **Later code path loads an additional native dependency (edge case)**
+ - SNI may be present, but a dependency touched by a later feature path is missing.
+ - Can look like "worked earlier, failed later" if paths differ.
+ - Evidence to collect: feature-specific repro matrix; compare auth/query paths.
+
+#### Most likely root-cause pattern
+
+The strongest fit for "works for a while, then fails" is delayed first load plus
+environment drift before the first database call:
+
+- Service starts and does non-database work.
+- SNI is not loaded yet (lazy native load path).
+- Temp cleanup, deployment drift, architecture mismatch exposure, or AV timing
+ changes the load environment.
+- First connection attempt happens later and fails with `0x8007007E`.
+
+This pattern maps directly to scenarios B, D, and E:
+
+- **B**: no matching-architecture SNI DLL is available for the process bitness.
+- **D**: single-file extracted native payload under `%TEMP%` is removed before first load.
+- **E**: AV/EDR transiently interferes with the file at load time.
+
+#### How to reason about scenario C in this symptom pattern
+
+Scenario C (missing transitive dependency such as VC++ runtime) is still possible,
+but it is usually consistent rather than intermittent. It appears "delayed" only
+when the first connection is delayed. Once the process has loaded SNI successfully,
+scenario C should not reappear inside that same process lifetime.
+
+#### Why this matters for reproductions
+
+Because SNI load is one-time per process, reproductions should focus on the first
+load window, not steady-state `Open()/Close()` loops after successful load.
+
+That is why the tooling in this folder separates:
+
+- load-window probes (`--probe-*` modes), and
+- load-time interference stress (`--stress-load-absent`).
+
+These modes are designed to diagnose first-load failures that surface as
+`0x8007007E`, including cases that appear as "long-running" in operational logs.