Merge pull request #221 from AzureCosmosDB/copilot/fix-date-format-migration-issue

Fix ISO-8601 date format preservation during data migration

Author: philnach

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension.UnitTests/CosmosDataSinkExtensionTests.cs

@@ -1,12 +1,80 @@
 using Cosmos.DataTransfer.Interfaces;
 using System.Dynamic;
+using System.Text;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Newtonsoft.Json;
 
 namespace Cosmos.DataTransfer.CosmosExtension.UnitTests
 {
     [TestClass]
     public class CosmosDataSinkExtensionTests
     {
+        [TestMethod]
+        public void CreateItemStream_WithDateString_PreservesFormat()
+        {
+            // Arrange - Simulate data read from source with ISO-8601 date string
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"event_time\": \"2023-12-19T00:00:00.000Z\"}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act - Create data item and build expando object (simulating the pipeline)
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            
+            // Serialize using the same settings that CreateItemStream uses
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert - The date string format should be preserved
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"Date format should be preserved. Actual JSON: {json}");
+        }
+
+        [TestMethod]
+        public void CreateItemStream_WithNestedDateString_PreservesFormat()
+        {
+            // Arrange - Simulate data with nested date strings
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"data\": {\"created\": \"2023-12-19T00:00:00.000Z\", \"modified\": \"2023-12-20T12:30:45.123Z\"}}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"Created date format should be preserved. Actual JSON: {json}");
+            Assert.IsTrue(json.Contains("\"2023-12-20T12:30:45.123Z\""), 
+                $"Modified date format should be preserved. Actual JSON: {json}");
+        }
+
+        [TestMethod]
+        public void CreateItemStream_WithDateStringArray_PreservesFormat()
+        {
+            // Arrange - Simulate data with date strings in array
+            var sourceSettings = RawJsonCosmosSerializer.GetDefaultSettings();
+            var sourceJson = "{\"id\": \"1\", \"timestamps\": [\"2023-12-19T00:00:00.000Z\", \"2023-12-20T00:00:00.000Z\"]}";
+            var serializer = JsonSerializer.Create(sourceSettings);
+            using var reader = new JsonTextReader(new StringReader(sourceJson));
+            var sourceDict = serializer.Deserialize<Dictionary<string, object?>>(reader)!;
+            
+            // Act
+            var dataItem = new CosmosDictionaryDataItem(sourceDict);
+            var expando = dataItem.BuildDynamicObjectTree()!;
+            var json = JsonConvert.SerializeObject(expando, RawJsonCosmosSerializer.GetDefaultSettings());
+            
+            // Assert
+            Assert.IsTrue(json.Contains("\"2023-12-19T00:00:00.000Z\""), 
+                $"First date format should be preserved. Actual JSON: {json}");
+            Assert.IsTrue(json.Contains("\"2023-12-20T00:00:00.000Z\""), 
+                $"Second date format should be preserved. Actual JSON: {json}");
+        }
+
         [TestMethod]
         public void BuildDynamicObjectTree_WithNestedArrays_WorksCorrectly()
         {

Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosDataSinkExtension.cs

@@ -280,7 +280,7 @@ private static async Task<ItemResult> PopulateItem(Container container, ExpandoO
 
         private static MemoryStream CreateItemStream(ExpandoObject item)
         {
-            var json = JsonConvert.SerializeObject(item);
+            var json = JsonConvert.SerializeObject(item, RawJsonCosmosSerializer.GetDefaultSettings());
             return new MemoryStream(Encoding.UTF8.GetBytes(json));
         }
 

Interfaces/Cosmos.DataTransfer.Common.UnitTests/DataItemJsonConverterTests.cs

@@ -64,6 +64,8 @@ public static IEnumerable<object?[]> Test_WriteFieldValue_Data { get
         yield return new object[] { false, "\"x\":false" };
         yield return new object[] { new DateTime(2025, 10, 15, 16, 45, 12, 666, DateTimeKind.Unspecified), "\"x\":\"2025-10-15T16:45:12.6660000\"" };
         yield return new object[] { new DateTime(2025, 10, 15, 16, 45, 12, 666, DateTimeKind.Utc), "\"x\":\"2025-10-15T16:45:12.6660000Z\"" };
+        yield return new object[] { new DateTimeOffset(2025, 10, 15, 16, 45, 12, 666, TimeSpan.Zero), "\"x\":\"2025-10-15T16:45:12.6660000+00:00\"" };
+        yield return new object[] { new DateTimeOffset(2025, 10, 15, 16, 45, 12, 666, TimeSpan.FromHours(5)), "\"x\":\"2025-10-15T16:45:12.6660000+05:00\"" };
         yield return new object[] { 'a', "\"x\":\"a\"" };
         yield return new object[] { "Greetings human    ", "\"x\":\"Greetings human    \"" };
         // yield return new object[] { true, "\"x\":true" };

Interfaces/Cosmos.DataTransfer.Common/DataItemJsonConverter.cs

@@ -139,7 +139,11 @@ internal static void WriteFieldValue(Utf8JsonWriter writer, string fieldName, ob
                     }
                     else if (arrayItem is DateTime date)
                     {
-                        writer.WriteStringValue(date.ToString("O"));
+                        writer.WriteStringValue(GetAsUnescaped(date.ToString("O")));
+                    }
+                    else if (arrayItem is DateTimeOffset dateOffset)
+                    {
+                        writer.WriteStringValue(GetAsUnescaped(dateOffset.ToString("O")));
                     }
                     else if (arrayItem is null)
                     {
@@ -178,7 +182,11 @@ internal static void WriteFieldValue(Utf8JsonWriter writer, string fieldName, ob
             }
             else if (fieldValue is DateTime date)
             {
-                writer.WriteString(propertyName, date.ToString("O"));
+                writer.WriteString(propertyName, GetAsUnescaped(date.ToString("O")));
+            }
+            else if (fieldValue is DateTimeOffset dateOffset)
+            {
+                writer.WriteString(propertyName, GetAsUnescaped(dateOffset.ToString("O")));
             }
             else
             {