From f8395e1272a749857a736b5b8521cf62de9f7f4f Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Tue, 3 Mar 2026 18:03:13 +0000 Subject: [PATCH 1/4] Add opt-in .NET HTTP diagnostics for Cosmos connections --- .../CertificateConfigurationTests.cs | 52 +++++++++++++++++++ .../CosmosExtensionServices.cs | 49 +++++++++++++++++ .../CosmosSettingsBase.cs | 6 +++ Extensions/Cosmos/README.md | 3 ++ 4 files changed, 110 insertions(+) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs index f88f080..8a7172a 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CertificateConfigurationTests.cs @@ -95,6 +95,58 @@ public void CreateClient_WithoutDisableSslValidation_DoesNotLogWarning() Assert.IsNotNull(client, "CosmosClient should be created"); } + [TestMethod] + public void CreateClient_WithEnableNetHttpLogging_LogsWarning() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + EnableNetHttpLogging = true + }; + + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains(".NET HTTP diagnostic logging is ENABLED")), + It.IsAny(), + It.IsAny>()), + Times.Once); + + Assert.IsNotNull(client, "CosmosClient should be created"); + } + + [TestMethod] + public void CreateClient_WithoutEnableNetHttpLogging_DoesNotLogHttpWarning() + { + var loggerMock = new Mock(); + var settings = new TestableCosmosSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + Database = "testDb", + Container = "testContainer", + EnableNetHttpLogging = false + }; + + var client = CosmosExtensionServices.CreateClient(settings, "TestDisplay", loggerMock.Object); + + loggerMock.Verify( + x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString()!.Contains(".NET HTTP diagnostic logging is ENABLED")), + It.IsAny(), + It.IsAny>()), + Times.Never); + + Assert.IsNotNull(client, "CosmosClient should be created"); + } + private class TestableCosmosSettings : CosmosSettingsBase { } diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index c158a08..7d7abce 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -9,6 +9,7 @@ using System.Net.Http; using System.Reflection; using System.Text.RegularExpressions; +using System.Diagnostics.Tracing; namespace Cosmos.DataTransfer.CosmosExtension { @@ -36,6 +37,9 @@ public static class CosmosExtensionServices return new HttpClient(handler); }); + private static readonly object _httpEventListenerLock = new(); + private static EventListener? _httpEventListener; + public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, ILogger logger, string? sourceDisplayName = null) { string userAgentString = CreateUserAgentString(displayName, sourceDisplayName); @@ -82,6 +86,12 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp logger.LogWarning("SSL certificate validation is DISABLED. This should ONLY be used for development scenarios. Never use in production."); clientOptions.ServerCertificateCustomValidationCallback = (cert, chain, errors) => true; } + + if (settings.EnableNetHttpLogging) + { + logger.LogWarning(".NET HTTP diagnostic logging is ENABLED for Cosmos connection troubleshooting. Logs are emitted at Debug level and may include endpoint/request details."); + EnsureHttpEventLoggingEnabled(logger); + } CosmosClient? cosmosClient; if (settings.UseRbacAuth) @@ -107,6 +117,45 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp return cosmosClient; } + private static void EnsureHttpEventLoggingEnabled(ILogger logger) + { + lock (_httpEventListenerLock) + { + _httpEventListener ??= new NetHttpEventListener(logger); + } + } + + private sealed class NetHttpEventListener(ILogger logger) : EventListener + { + private readonly ILogger _logger = logger; + + protected override void OnEventSourceCreated(EventSource eventSource) + { + if (eventSource.Name == "System.Net.Http" || eventSource.Name == "System.Net.Security") + { + EnableEvents(eventSource, EventLevel.Informational, EventKeywords.All); + } + } + + protected override void OnEventWritten(EventWrittenEventArgs eventData) + { + if (!_logger.IsEnabled(LogLevel.Debug)) + { + return; + } + + var payload = eventData.Payload?.Count > 0 + ? string.Join(", ", eventData.Payload!.Select(v => v?.ToString() ?? "null")) + : string.Empty; + + _logger.LogDebug(".NET HTTP event {Source}:{EventName} ({EventId}) {Payload}", + eventData.EventSource.Name, + eventData.EventName ?? "unknown", + eventData.EventId, + payload); + } + } + private static string CreateUserAgentString(string displayName, string? sourceDisplayName) { // based on: diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index d147ec0..0f6e98f 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -47,6 +47,12 @@ public abstract class CosmosSettingsBase : IValidatableObject /// public bool AllowBulkExecution { get; set; } = false; + /// + /// Enables .NET HTTP diagnostic event logging for troubleshooting connectivity issues. + /// Logs are emitted at Debug level and may include endpoint and request details. + /// + public bool EnableNetHttpLogging { get; set; } = false; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 58b68bf..ce53ada 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -44,6 +44,7 @@ These properties will be preserved exactly as they appear in the source when mig | LimitToEndpoint | Restrict client to endpoint (see CosmosClientOptions.LimitToEndpoint) | false | | DisableSslValidation | Disable SSL certificate validation (for local dev only; not for production) | false | | AllowBulkExecution | Enable bulk execution for optimized performance.
**Warning:** May affect consistency and error handling. | false | +| EnableNetHttpLogging | Enable .NET HTTP diagnostic events for troubleshooting connectivity issues (Debug log level) | false | Source and sink require settings used to locate and access the Cosmos DB account. This can be done in one of two ways: @@ -79,6 +80,7 @@ Source supports the following optional parameters: - `UseDefaultProxyCredentials` (`false` by default) - When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks). - `UseDefaultCredentials` (`false` by default) - When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy. - `PreAuthenticate` (`false` by default) - When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. +- `EnableNetHttpLogging` (`false` by default) - When `true`, enables `.NET` `System.Net.Http` diagnostic events to help investigate connection issues. These events are emitted at Debug level and can include endpoint/request details. ### Always Encrypted @@ -175,6 +177,7 @@ For development purposes with SSL validation disabled: - **`UseDefaultProxyCredentials`**: Optional, defaults to `false`. When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks). - **`UseDefaultCredentials`**: Optional, defaults to `false`. When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy. - **`PreAuthenticate`**: Optional, defaults to `false`. When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. +- **`EnableNetHttpLogging`**: Optional, defaults to `false`. When `true`, enables `.NET` `System.Net.Http` diagnostic events for troubleshooting. Events are logged at Debug level and may include endpoint/request details. - **`LimitToEndpoint`**: Optional, defaults to `false`. When the value of this property is false, the Cosmos DB SDK will automatically discover write and read regions, and use them when the configured application region is not available. From 8b533e2837c5319ce37d7784678aa11ebb041b51 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 4 Mar 2026 09:50:29 +0000 Subject: [PATCH 2/4] Add more details to readme --- Extensions/Cosmos/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index ce53ada..c128dfd 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -82,6 +82,28 @@ Source supports the following optional parameters: - `PreAuthenticate` (`false` by default) - When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. - `EnableNetHttpLogging` (`false` by default) - When `true`, enables `.NET` `System.Net.Http` diagnostic events to help investigate connection issues. These events are emitted at Debug level and can include endpoint/request details. +### Connectivity troubleshooting (.NET HTTP diagnostics) + +When connection failures are not verbose enough, enable `EnableNetHttpLogging` and run the tool with `Debug` logging. + +```json +{ + "SourceSettings": { + "ConnectionMode": "Gateway", + "LimitToEndpoint": true, + "DisableSslValidation": true, + "EnableNetHttpLogging": true + } +} +``` + +Set logging to Debug before running: + +- Linux/macOS: `export Logging__LogLevel__Default=Debug` +- Windows (PowerShell): `$env:Logging__LogLevel__Default = "Debug"` + +This emits `System.Net.Http` and `System.Net.Security` events at Debug level, which helps distinguish TLS/certificate failures from transport/proxy/network failures. + ### Always Encrypted Source and Sink support Always Encrypted as an optional parameter. When `InitClientEncryption` is set to `true`, the extension will initialize the Cosmos client with the Always Encrypted feature enabled. This allows for the use of encrypted fields in the Cosmos DB container. The extension will automatically decrypt the fields when reading from the source and encrypt the fields when writing to the sink. From 731785458ccaa433020b297cacff1b5c55c8c9cc Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 4 Mar 2026 10:54:37 +0000 Subject: [PATCH 3/4] change ordering to common .NET app configuration order --- Core/Cosmos.DataTransfer.Core/Program.cs | 25 +++++++++++++++--------- Extensions/Cosmos/README.md | 3 +++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Core/Cosmos.DataTransfer.Core/Program.cs b/Core/Cosmos.DataTransfer.Core/Program.cs index d290a86..90c8740 100644 --- a/Core/Cosmos.DataTransfer.Core/Program.cs +++ b/Core/Cosmos.DataTransfer.Core/Program.cs @@ -52,18 +52,25 @@ public static async Task Main(string[] args) { builder.ConfigureAppConfiguration((hostContext, cfg) => { + // Keep explicit .NET precedence order: + // appsettings.json -> appsettings.{Environment}.json (for example appsettings.Development.json) + // -> user secrets (only when Environment == Development) + // -> environment variables -> command line. + // This ensures env vars and CLI args override file-based defaults. + cfg.Sources.Clear(); + var exeFolder = AppContext.BaseDirectory; - var appsettings = Path.Combine(exeFolder, "appsettings.json"); - if (File.Exists(appsettings)) - { - cfg.AddJsonFile(appsettings); - } - var appsettingsEnv = Path.Combine(exeFolder, $"appsettings.{hostContext.HostingEnvironment.EnvironmentName}.json"); - if (File.Exists(appsettingsEnv)) + cfg.SetBasePath(exeFolder); + cfg.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false); + cfg.AddJsonFile($"appsettings.{hostContext.HostingEnvironment.EnvironmentName}.json", optional: true, reloadOnChange: false); + + if (hostContext.HostingEnvironment.IsDevelopment()) { - cfg.AddJsonFile(appsettingsEnv); + cfg.AddUserSecrets(optional: true); } - cfg.AddUserSecrets(); + + cfg.AddEnvironmentVariables(); + cfg.AddCommandLine(args); }).ConfigureServices((hostContext, services) => { services.AddTransient(); diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index c128dfd..a5742a4 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -100,9 +100,12 @@ When connection failures are not verbose enough, enable `EnableNetHttpLogging` a Set logging to Debug before running: - Linux/macOS: `export Logging__LogLevel__Default=Debug` +- Windows (cmd): `set Logging__LogLevel__Default=Debug` - Windows (PowerShell): `$env:Logging__LogLevel__Default = "Debug"` This emits `System.Net.Http` and `System.Net.Security` events at Debug level, which helps distinguish TLS/certificate failures from transport/proxy/network failures. +Configuration follows normal .NET precedence, so environment variables override `appsettings.json` values. +Command-line arguments have highest precedence and override both environment variables and `appsettings.json`. ### Always Encrypted From c2602c385f0556650bb36feda20873ef029e8bd3 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 4 Mar 2026 11:00:17 +0000 Subject: [PATCH 4/4] Remove Event Counters message --- .../CosmosExtensionServices.cs | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index 7d7abce..ea5b145 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -144,9 +144,17 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) return; } - var payload = eventData.Payload?.Count > 0 - ? string.Join(", ", eventData.Payload!.Select(v => v?.ToString() ?? "null")) - : string.Empty; + if (eventData.EventSource.Name != "System.Net.Http" && eventData.EventSource.Name != "System.Net.Security") + { + return; + } + + if (string.Equals(eventData.EventName, "EventCounters", StringComparison.Ordinal)) + { + return; + } + + var payload = FormatPayload(eventData); _logger.LogDebug(".NET HTTP event {Source}:{EventName} ({EventId}) {Payload}", eventData.EventSource.Name, @@ -154,6 +162,42 @@ protected override void OnEventWritten(EventWrittenEventArgs eventData) eventData.EventId, payload); } + + private static string FormatPayload(EventWrittenEventArgs eventData) + { + if (eventData.Payload is null || eventData.Payload.Count == 0) + { + return string.Empty; + } + + return string.Join(", ", eventData.Payload.Select((value, index) => + { + var key = eventData.PayloadNames is not null && index < eventData.PayloadNames.Count + ? eventData.PayloadNames[index] + : $"arg{index}"; + return $"{key}={FormatPayloadValue(value)}"; + })); + } + + private static string FormatPayloadValue(object? value) + { + if (value is null) + { + return "null"; + } + + if (value is string text) + { + return text; + } + + if (value is IEnumerable> keyValuePairs) + { + return string.Join(";", keyValuePairs.Select(kvp => $"{kvp.Key}={kvp.Value}")); + } + + return value.ToString() ?? string.Empty; + } } private static string CreateUserAgentString(string displayName, string? sourceDisplayName)