Merge branch 'main' into vnextemulator

philnach · web-flow · commit 9dd7437ec997 · 2025-12-05T05:30:51.000-08:00
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Release Binaries ${{ github.event.inputs.release-tag }}
+name: Release desktop binaries
 on: 
   workflow_dispatch:
     inputs:
diff --git a/ExampleConfigs.md b/ExampleConfigs.md
@@ -202,3 +202,26 @@
   ]
 }
 ```
+
+## JSON to Cosmos-NoSQL (Using Authenticated Proxy)
+
+```json
+{
+    "Source": "json",
+    "Sink": "cosmos-nosql",
+    "SourceSettings": {
+        "FilePath": "c:\\data\\sales-data.json"
+    },
+    "SinkSettings": {
+        "ConnectionString": "AccountEndpoint=https://...",
+        "Database": "myDb",
+        "Container": "myContainer",
+        "PartitionKeyPath": "/id",
+        "WriteMode": "Insert",
+        "WebProxy": "http://yourproxy.server.com/",
+        "UseDefaultProxyCredentials": true,
+        "UseDefaultCredentials": true,
+        "PreAuthenticate": true
+    }
+}
+```
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs
@@ -1,12 +1,80 @@
 ﻿using Cosmos.DataTransfer.Interfaces;
 using System.Dynamic;
+using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Newtonsoft.Json;
 
 namespace Cosmos.DataTransfer.CosmosExtension.UnitTests
 {
     [TestClass]
     public class CosmosDataSinkExtensionTests
     {
+        [TestMethod]
+        public void CreateItemStream_WithDateString_PreservesFormat()
+        {
+            // Arrange - Simulate data read from source with ISO-8601 date string
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"event_time\": \"2023-12-19T00:00:00.000Z\"}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act - Create data item and build expando object (simulating the pipeline)
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            
+            // Serialize using the same settings that CreateItemStream uses
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert - The date string format should be preserved
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"Date format should be preserved. Actual JSON: {json}");
+        }
+
+        [TestMethod]
+        public void CreateItemStream_WithNestedDateString_PreservesFormat()
+        {
+            // Arrange - Simulate data with nested date strings
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"data\": {\"created\": \"2023-12-19T00:00:00.000Z\", \"modified\": \"2023-12-20T12:30:45.123Z\"}}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"Created date format should be preserved. Actual JSON: {json}");
+            Assert.IsTrue(json.Contains("\"2023-12-20T12:30:45.123Z\""), 
+                $"Modified date format should be preserved. Actual JSON: {json}");
+        }
+
+        [TestMethod]
+        public void CreateItemStream_WithDateStringArray_PreservesFormat()
+        {
+            // Arrange - Simulate data with date strings in array
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"timestamps\": [\"2023-12-19T00:00:00.000Z\", \"2023-12-20T00:00:00.000Z\"]}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"First date format should be preserved. Actual JSON: {json}");
+            Assert.IsTrue(json.Contains("\"2023-12-20T00:00:00.000Z\""), 
+                $"Second date format should be preserved. Actual JSON: {json}");
+        }
+
         [TestMethod]
         public void BuildDynamicObjectTree_WithNestedArrays_WorksCorrectly()
         {
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSinkSettingsTests.cs
@@ -209,4 +209,85 @@ public void GetValidationErrors_WhenRbacWithoutInitClientEncryption_Succeeds()
 
         Assert.AreEqual(0, validationErrors.Count());
     }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultProxyCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_DefaultsToFalse()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.PreAuthenticate);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_CanBeSetToTrue()
+    {
+        var settings = new CosmosSinkSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            PreAuthenticate = true,
+        };
+
+        Assert.IsTrue(settings.PreAuthenticate);
+    }
 }
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosSourceSettingsTests.cs
@@ -60,4 +60,85 @@ public void Validate_WithAccountEndpoint_Succeeds()
 
         settings.Validate();
     }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultProxyCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultProxyCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultProxyCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void UseDefaultCredentials_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            UseDefaultCredentials = true,
+        };
+
+        Assert.IsTrue(settings.UseDefaultCredentials);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_DefaultsToFalse()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+        };
+
+        Assert.IsFalse(settings.PreAuthenticate);
+    }
+
+    [TestMethod]
+    public void PreAuthenticate_CanBeSetToTrue()
+    {
+        var settings = new CosmosSourceSettings
+        {
+            ConnectionString = "AccountEndpoint=https://localhost:8081/;AccountKey=",
+            Database = "db",
+            Container = "container",
+            PreAuthenticate = true,
+        };
+
+        Assert.IsTrue(settings.PreAuthenticate);
+    }
 }
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs
@@ -280,7 +280,7 @@ private static async Task<ItemResult> PopulateItem(Container container, ExpandoO
 
         private static MemoryStream CreateItemStream(ExpandoObject item)
         {
-            var json = JsonConvert.SerializeObject(item);
+            var json = JsonConvert.SerializeObject(item, RawJsonCosmosSerializer.GetDefaultSettings());
             return new MemoryStream(Encoding.UTF8.GetBytes(json));
         }
 
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosExtensionServices.cs
@@ -6,13 +6,36 @@
 using Microsoft.Extensions.Logging;
 using System.Globalization;
 using System.Net;
+using System.Net.Http;
 using System.Reflection;
 using System.Text.RegularExpressions;
 
 namespace Cosmos.DataTransfer.CosmosExtension
 {
     public static class CosmosExtensionServices
     {
+        // Static HttpClient instances with different configurations for reuse across connections
+        // This avoids connection exhaustion and properly handles credentials
+        private static readonly Lazy<HttpClient> _httpClientWithDefaultCredentials = new Lazy<HttpClient>(() =>
+        {
+            var handler = new HttpClientHandler
+            {
+                Credentials = CredentialCache.DefaultNetworkCredentials,
+                PreAuthenticate = false
+            };
+            return new HttpClient(handler);
+        });
+
+        private static readonly Lazy<HttpClient> _httpClientWithDefaultCredentialsAndPreAuth = new Lazy<HttpClient>(() =>
+        {
+            var handler = new HttpClientHandler
+            {
+                Credentials = CredentialCache.DefaultNetworkCredentials,
+                PreAuthenticate = true
+            };
+            return new HttpClient(handler);
+        });
+
         public static CosmosClient CreateClient(CosmosSettingsBase settings, string displayName, ILogger logger, string? sourceDisplayName = null)
         {
             string userAgentString = CreateUserAgentString(displayName, sourceDisplayName);
@@ -36,7 +59,21 @@ public static CosmosClient CreateClient(CosmosSettingsBase settings, string disp
             };
 
             if (!string.IsNullOrEmpty(settings.WebProxy)){
-                clientOptions.WebProxy = new WebProxy(settings.WebProxy);
+                var webProxy = new WebProxy(settings.WebProxy);
+                if (settings.UseDefaultProxyCredentials)
+                {
+                    webProxy.UseDefaultCredentials = true;
+                }
+                clientOptions.WebProxy = webProxy;
+            }
+
+            // Configure the HttpClient with default credentials if requested
+            // This enables authenticated proxy support for the underlying HTTP connections
+            if (settings.UseDefaultCredentials)
+            {
+                clientOptions.HttpClientFactory = settings.PreAuthenticate
+                    ? () => _httpClientWithDefaultCredentialsAndPreAuth.Value
+                    : () => _httpClientWithDefaultCredentials.Value;
             }
 
             // Disable SSL certificate validation for development scenarios
diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs
@@ -12,6 +12,9 @@ public abstract class CosmosSettingsBase : IValidatableObject
         public string? Container { get; set; }
         public ConnectionMode ConnectionMode { get; set; } = ConnectionMode.Gateway;
         public string? WebProxy { get; set; }
+        public bool UseDefaultProxyCredentials { get; set; } = false;
+        public bool UseDefaultCredentials { get; set; } = false;
+        public bool PreAuthenticate { get; set; } = false;
         public bool UseRbacAuth { get; set; }
         public string? AccountEndpoint { get; set; }
         public bool EnableInteractiveCredentials { get; set; }
diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md
diff --git a/Interfaces/Cosmos.DataTransfer.Common.UnitTests/DataItemJsonConverterTests.cs b/Interfaces/Cosmos.DataTransfer.Common.UnitTests/DataItemJsonConverterTests.cs
diff --git a/Interfaces/Cosmos.DataTransfer.Common/DataItemJsonConverter.cs b/Interfaces/Cosmos.DataTransfer.Common/DataItemJsonConverter.cs

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-name: Release Binaries ${{ github.event.inputs.release-tag }}`
	`1`	`+name: Release desktop binaries`
`2`	`2`	`on:`
`3`	`3`	`workflow_dispatch:`
`4`	`4`	`inputs:`
Original file line number	Diff line number	Diff line change
`@@ -280,7 +280,7 @@ private static async Task<ItemResult> PopulateItem(Container container, ExpandoO`
`280`	`280`
`281`	`281`	`private static MemoryStream CreateItemStream(ExpandoObject item)`
`282`	`282`	`{`
`283`		`- var json = JsonConvert.SerializeObject(item);`
	`283`	`+ var json = JsonConvert.SerializeObject(item, RawJsonCosmosSerializer.GetDefaultSettings());`
`284`	`284`	`return new MemoryStream(Encoding.UTF8.GetBytes(json));`
`285`	`285`	`}`
`286`	`286`