Merge pull request #220 from AzureCosmosDB/copilot/support-authenticated-proxies

Support authenticated proxies with separate settings for WebProxy, HttpClient credentials, and PreAuthentication

Author: markjbrown

ExampleConfigs.md

@@ -202,3 +202,26 @@
   ]
 }
 ```
+
+## JSON to Cosmos-NoSQL (Using Authenticated Proxy)
+
+```json
+{
+    "Source": "json",
+    "Sink": "cosmos-nosql",
+    "SourceSettings": {
+        "FilePath": "c:\\data\\sales-data.json"
+    },
+    "SinkSettings": {
+        "ConnectionString": "AccountEndpoint=https://...",
+        "Database": "myDb",
+        "Container": "myContainer",
+        "PartitionKeyPath": "/id",
+        "WriteMode": "Insert",
+        "WebProxy": "http://yourproxy.server.com/",
+        "UseDefaultProxyCredentials": true,
+        "UseDefaultCredentials": true,
+        "PreAuthenticate": true
+    }
+}
+```

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs

@@ -209,4 +209,85 @@ public void GetValidationErrors_WhenRbacWithoutInitClientEncryption_Succeeds()
 
         Assert.AreEqual(0, validationErrors.Count());
     }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultProxyCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.PreAuthenticate);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            PreAuthenticate = true,
+        };
+
+        Assert.IsTrue(settings.PreAuthenticate);
+    }
 }

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs

@@ -60,4 +60,85 @@ public void Validate_WithAccountEndpoint_Succeeds()
 
         settings.Validate();
     }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultProxyCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.PreAuthenticate);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            PreAuthenticate = true,
+        };
+
+        Assert.IsTrue(settings.PreAuthenticate);
+    }
 }

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs

@@ -9,11 +9,34 @@
 using Microsoft.Azure.Cosmos.Encryption;
 using Azure.Security.KeyVault.Keys.Cryptography;
 using System.Net;
+using System.Net.Http;
 
 namespace Cosmos.DataTransfer.CosmosExtension
 {
     public static class CosmosExtensionServices
     {
+        // Static HttpClient instances with different configurations for reuse across connections
+        // This avoids connection exhaustion and properly handles credentials
+        private static readonly Lazy<HttpClient> _httpClientWithDefaultCredentials = new Lazy<HttpClient>(() =>
+        {
+            var handler = new HttpClientHandler
+            {
+                Credentials = CredentialCache.DefaultNetworkCredentials,
+                PreAuthenticate = false
+            };
+            return new HttpClient(handler);
+        });
+
+        private static readonly Lazy<HttpClient> _httpClientWithDefaultCredentialsAndPreAuth = new Lazy<HttpClient>(() =>
+        {
+            var handler = new HttpClientHandler
+            {
+                Credentials = CredentialCache.DefaultNetworkCredentials,
+                PreAuthenticate = true
+            };
+            return new HttpClient(handler);
+        });
+
         public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, string? sourceDisplayName = null)
         {
             string userAgentString = CreateUserAgentString(displayName, sourceDisplayName);
@@ -37,7 +60,21 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp
             };
 
             if (!string.IsNullOrEmpty(settings.WebProxy)){
-                clientOptions.WebProxy = new WebProxy(settings.WebProxy);
+                var webProxy = new WebProxy(settings.WebProxy);
+                if (settings.UseDefaultProxyCredentials)
+                {
+                    webProxy.UseDefaultCredentials = true;
+                }
+                clientOptions.WebProxy = webProxy;
+            }
+
+            // Configure the HttpClient with default credentials if requested
+            // This enables authenticated proxy support for the underlying HTTP connections
+            if (settings.UseDefaultCredentials)
+            {
+                clientOptions.HttpClientFactory = settings.PreAuthenticate
+                    ? () => _httpClientWithDefaultCredentialsAndPreAuth.Value
+                    : () => _httpClientWithDefaultCredentials.Value;
             }
 
             CosmosClient? cosmosClient;

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs

@@ -12,6 +12,9 @@ public abstract class CosmosSettingsBase : IValidatableObject
         public string? Container { get; set; }
         public ConnectionMode ConnectionMode { get; set; } = ConnectionMode.Gateway;
         public string? WebProxy { get; set; }
+        public bool UseDefaultProxyCredentials { get; set; } = false;
+        public bool UseDefaultCredentials { get; set; } = false;
+        public bool PreAuthenticate { get; set; } = false;
         public bool UseRbacAuth { get; set; }
         public string? AccountEndpoint { get; set; }
         public bool EnableInteractiveCredentials { get; set; }

Extensions/Cosmos/README.md

@@ -16,7 +16,14 @@ Source and sink settings also both require parameters to specify the data locati
 - `Database`
 - `Container`
 
-Source supports an optional `IncludeMetadataFields` parameter (`false` by default) to enable inclusion of built-in Cosmos fields prefixed with `"_"`, for example `"_etag"` and `"_ts"`. An optional PartitionKeyValue setting allows for filtering to a single partition. The optional Query setting allows further filtering using a Cosmos SQL statement. An optional `WebProxy` parameter (`null` by default) enables connections through a proxy.
+Source supports the following optional parameters:
+- `IncludeMetadataFields` (`false` by default) - Enables inclusion of built-in Cosmos fields prefixed with `"_"`, for example `"_etag"` and `"_ts"`.
+- `PartitionKeyValue` - Allows for filtering to a single partition.
+- `Query` - Allows further filtering using a Cosmos SQL statement.
+- `WebProxy` (`null` by default) - Enables connections through a proxy.
+- `UseDefaultProxyCredentials` (`false` by default) - When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks).
+- `UseDefaultCredentials` (`false` by default) - When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy.
+- `PreAuthenticate` (`false` by default) - When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted.
 
 ### Always Encrypted
 
@@ -36,7 +43,10 @@ The extension will also automatically handle the encryption keys and encryption
     "IncludeMetadataFields": false,
     "PartitionKeyValue":"123",
     "Query":"SELECT * FROM c WHERE c.category='event'",
-    "WebProxy":"http://yourproxy.server.com/"
+    "WebProxy":"http://yourproxy.server.com/",
+    "UseDefaultProxyCredentials": true,
+    "UseDefaultCredentials": true,
+    "PreAuthenticate": true
 }
 ```
 
@@ -52,8 +62,11 @@ Or with RBAC:
     "IncludeMetadataFields": false,
     "PartitionKeyValue":"123",
     "Query":"SELECT * FROM c WHERE c.category='event'",
-    "InitClientEncryption": false
-    "WebProxy":"http://yourproxy.server.com/"
+    "InitClientEncryption": false,
+    "WebProxy":"http://yourproxy.server.com/",
+    "UseDefaultProxyCredentials": true,
+    "UseDefaultCredentials": true,
+    "PreAuthenticate": true
 }
 ```
 
@@ -85,6 +98,11 @@ Or with RBAC:
   - `Gateway` (default)
   - `Direct`
 
+- **`WebProxy`**: Optional. Specifies the proxy server URL to use for connections (e.g., `http://yourproxy.server.com/`).
+- **`UseDefaultProxyCredentials`**: Optional, defaults to `false`. When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks).
+- **`UseDefaultCredentials`**: Optional, defaults to `false`. When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy.
+- **`PreAuthenticate`**: Optional, defaults to `false`. When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted.
+
 - **`LimitToEndpoint`**: Optional, defaults to `false`. When the value of this property is false, the Cosmos DB SDK will automatically discover
   write and read regions, and use them when the configured application region is not available. 
   When set to `true`, availability is limited to the endpoint specified.