diff --git a/ExampleConfigs.md b/ExampleConfigs.md index 6007bd0..d67810c 100644 --- a/ExampleConfigs.md +++ b/ExampleConfigs.md @@ -202,3 +202,26 @@ ] } ``` + +## JSON to Cosmos-NoSQL (Using Authenticated Proxy) + +```json +{ + "Source": "json", + "Sink": "cosmos-nosql", + "SourceSettings": { + "FilePath": "c:\\data\\sales-data.json" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://...", + "Database": "myDb", + "Container": "myContainer", + "PartitionKeyPath": "/id", + "WriteMode": "Insert", + "WebProxy": "http://yourproxy.server.com/", + "UseDefaultProxyCredentials": true, + "UseDefaultCredentials": true, + "PreAuthenticate": true + } +} +``` diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs index 68cbeab..f724911 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs @@ -209,4 +209,85 @@ public void GetValidationErrors_WhenRbacWithoutInitClientEncryption_Succeeds() Assert.AreEqual(0, validationErrors.Count()); } + + [TestMethod] + public void UseDefaultProxyCredentials_DefaultsToFalse() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.UseDefaultProxyCredentials); + } + + [TestMethod] + public void UseDefaultProxyCredentials_CanBeSetToTrue() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + UseDefaultProxyCredentials = true, + }; + + Assert.IsTrue(settings.UseDefaultProxyCredentials); + } + + [TestMethod] + public void UseDefaultCredentials_DefaultsToFalse() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.UseDefaultCredentials); + } + + [TestMethod] + public void UseDefaultCredentials_CanBeSetToTrue() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + UseDefaultCredentials = true, + }; + + Assert.IsTrue(settings.UseDefaultCredentials); + } + + [TestMethod] + public void PreAuthenticate_DefaultsToFalse() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.PreAuthenticate); + } + + [TestMethod] + public void PreAuthenticate_CanBeSetToTrue() + { + var settings = new CosmosSinkSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + PreAuthenticate = true, + }; + + Assert.IsTrue(settings.PreAuthenticate); + } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs index 3b894e1..1cb18f4 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs @@ -60,4 +60,85 @@ public void Validate_WithAccountEndpoint_Succeeds() settings.Validate(); } + + [TestMethod] + public void UseDefaultProxyCredentials_DefaultsToFalse() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.UseDefaultProxyCredentials); + } + + [TestMethod] + public void UseDefaultProxyCredentials_CanBeSetToTrue() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + UseDefaultProxyCredentials = true, + }; + + Assert.IsTrue(settings.UseDefaultProxyCredentials); + } + + [TestMethod] + public void UseDefaultCredentials_DefaultsToFalse() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.UseDefaultCredentials); + } + + [TestMethod] + public void UseDefaultCredentials_CanBeSetToTrue() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + UseDefaultCredentials = true, + }; + + Assert.IsTrue(settings.UseDefaultCredentials); + } + + [TestMethod] + public void PreAuthenticate_DefaultsToFalse() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + }; + + Assert.IsFalse(settings.PreAuthenticate); + } + + [TestMethod] + public void PreAuthenticate_CanBeSetToTrue() + { + var settings = new CosmosSourceSettings + { + ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=", + Database = "db", + Container = "container", + PreAuthenticate = true, + }; + + Assert.IsTrue(settings.PreAuthenticate); + } } \ No newline at end of file diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs index dd9729f..ba585ca 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs @@ -9,11 +9,34 @@ using Microsoft.Azure.Cosmos.Encryption; using Azure.Security.KeyVault.Keys.Cryptography; using System.Net; +using System.Net.Http; namespace Cosmos.DataTransfer.CosmosExtension { public static class CosmosExtensionServices { + // Static HttpClient instances with different configurations for reuse across connections + // This avoids connection exhaustion and properly handles credentials + private static readonly Lazy _httpClientWithDefaultCredentials = new Lazy(() => + { + var handler = new HttpClientHandler + { + Credentials = CredentialCache.DefaultNetworkCredentials, + PreAuthenticate = false + }; + return new HttpClient(handler); + }); + + private static readonly Lazy _httpClientWithDefaultCredentialsAndPreAuth = new Lazy(() => + { + var handler = new HttpClientHandler + { + Credentials = CredentialCache.DefaultNetworkCredentials, + PreAuthenticate = true + }; + return new HttpClient(handler); + }); + public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, string? sourceDisplayName = null) { string userAgentString = CreateUserAgentString(displayName, sourceDisplayName); @@ -37,7 +60,21 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp }; if (!string.IsNullOrEmpty(settings.WebProxy)){ - clientOptions.WebProxy = new WebProxy(settings.WebProxy); + var webProxy = new WebProxy(settings.WebProxy); + if (settings.UseDefaultProxyCredentials) + { + webProxy.UseDefaultCredentials = true; + } + clientOptions.WebProxy = webProxy; + } + + // Configure the HttpClient with default credentials if requested + // This enables authenticated proxy support for the underlying HTTP connections + if (settings.UseDefaultCredentials) + { + clientOptions.HttpClientFactory = settings.PreAuthenticate + ? () => _httpClientWithDefaultCredentialsAndPreAuth.Value + : () => _httpClientWithDefaultCredentials.Value; } CosmosClient? cosmosClient; diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index a81fac4..c381479 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -12,6 +12,9 @@ public abstract class CosmosSettingsBase : IValidatableObject public string? Container { get; set; } public ConnectionMode ConnectionMode { get; set; } = ConnectionMode.Gateway; public string? WebProxy { get; set; } + public bool UseDefaultProxyCredentials { get; set; } = false; + public bool UseDefaultCredentials { get; set; } = false; + public bool PreAuthenticate { get; set; } = false; public bool UseRbacAuth { get; set; } public string? AccountEndpoint { get; set; } public bool EnableInteractiveCredentials { get; set; } diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 82ea46f..ba3d519 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -16,7 +16,14 @@ Source and sink settings also both require parameters to specify the data locati - `Database` - `Container` -Source supports an optional `IncludeMetadataFields` parameter (`false` by default) to enable inclusion of built-in Cosmos fields prefixed with `"_"`, for example `"_etag"` and `"_ts"`. An optional PartitionKeyValue setting allows for filtering to a single partition. The optional Query setting allows further filtering using a Cosmos SQL statement. An optional `WebProxy` parameter (`null` by default) enables connections through a proxy. +Source supports the following optional parameters: +- `IncludeMetadataFields` (`false` by default) - Enables inclusion of built-in Cosmos fields prefixed with `"_"`, for example `"_etag"` and `"_ts"`. +- `PartitionKeyValue` - Allows for filtering to a single partition. +- `Query` - Allows further filtering using a Cosmos SQL statement. +- `WebProxy` (`null` by default) - Enables connections through a proxy. +- `UseDefaultProxyCredentials` (`false` by default) - When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks). +- `UseDefaultCredentials` (`false` by default) - When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy. +- `PreAuthenticate` (`false` by default) - When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. ### Always Encrypted @@ -36,7 +43,10 @@ The extension will also automatically handle the encryption keys and encryption "IncludeMetadataFields": false, "PartitionKeyValue":"123", "Query":"SELECT * FROM c WHERE c.category='event'", - "WebProxy":"http://yourproxy.server.com/" + "WebProxy":"http://yourproxy.server.com/", + "UseDefaultProxyCredentials": true, + "UseDefaultCredentials": true, + "PreAuthenticate": true } ``` @@ -52,8 +62,11 @@ Or with RBAC: "IncludeMetadataFields": false, "PartitionKeyValue":"123", "Query":"SELECT * FROM c WHERE c.category='event'", - "InitClientEncryption": false - "WebProxy":"http://yourproxy.server.com/" + "InitClientEncryption": false, + "WebProxy":"http://yourproxy.server.com/", + "UseDefaultProxyCredentials": true, + "UseDefaultCredentials": true, + "PreAuthenticate": true } ``` @@ -85,6 +98,11 @@ Or with RBAC: - `Gateway` (default) - `Direct` +- **`WebProxy`**: Optional. Specifies the proxy server URL to use for connections (e.g., `http://yourproxy.server.com/`). +- **`UseDefaultProxyCredentials`**: Optional, defaults to `false`. When `true`, includes default credentials in the WebProxy request. Use this when connecting through an authenticated proxy that returns [`407 Proxy Authentication Required`](https://learn.microsoft.com/dotnet/api/system.net.webproxy.credentials?view=net-10.0#remarks). +- **`UseDefaultCredentials`**: Optional, defaults to `false`. When `true`, configures the underlying HttpClient with default network credentials. Use this when the connection to CosmosDB requires authentication through a proxy. +- **`PreAuthenticate`**: Optional, defaults to `false`. When `true`, enables pre-authentication on the HttpClient, which sends credentials with the initial request rather than waiting for a 401/407 challenge. This can save extra round-trips but should only be used when the endpoint is trusted. + - **`LimitToEndpoint`**: Optional, defaults to `false`. When the value of this property is false, the Cosmos DB SDK will automatically discover write and read regions, and use them when the configured application region is not available. When set to `true`, availability is limited to the endpoint specified.