diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs index f57fc966c01..bd11f7ad464 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -319,8 +319,8 @@ public IList ReplaceImageGenerationFunctionResults(IList c { List? newContents = null; - // Replace FunctionResultContent instances with generated image content - for (int i = contents.Count - 1; i >= 0; i--) + // Replace image-generation FunctionCallContent with ImageGenerationToolCallContent, and FunctionResultContent with generated image content + for (int i = 0; i < contents.Count; i++) { var content = contents[i]; @@ -328,7 +328,7 @@ public IList ReplaceImageGenerationFunctionResults(IList c if (content is FunctionCallContent functionCall && _toolNames.Contains(functionCall.Name)) { - // create a new list and omit the FunctionCallContent + // create a new list copying all items before this one, and omit the FunctionCallContent newContents ??= CopyList(contents, i); if (functionCall.Name != nameof(GetImagesForEdit)) @@ -339,6 +339,7 @@ public IList ReplaceImageGenerationFunctionResults(IList c else if (content is FunctionResultContent functionResult && _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) { + // create a new list copying all items before this one newContents ??= CopyList(contents, i); if (imageContents.Any()) diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs index 0571b06d9fc..f8bc3ee1974 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs @@ -383,4 +383,152 @@ async IAsyncEnumerable GetUpdatesAsync() var imageToolCallContent = Assert.IsType(update.Contents[0]); Assert.Equal(callId, imageToolCallContent.CallId); } + + [Theory] + [InlineData("at-start")] + [InlineData("in-middle")] + [InlineData("at-end")] + public async Task GetResponseAsync_FunctionCallContent_SurroundingContentPreservedInOrder(string position) + { + // Regression test for: image generation content duplicating preceding content and dropping following content. + // The image-generation FunctionCallContent should be replaced with ImageGenerationToolCallContent, + // while all other content items retain their original order and cardinality. + var imageCallId = "image-call-id"; + var otherCallId = "other-call-id"; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + IList contents = position switch + { + "at-start" => [new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" }), + new FunctionResultContent(otherCallId, "other-result")], + "at-end" => [new FunctionResultContent(otherCallId, "other-result"), + new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" })], + _ => [new FunctionResultContent(otherCallId, "before-result"), + new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" }), + new UsageContent(new UsageDetails { InputTokenCount = 5 })], + }; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, contents))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.NotNull(response); + Assert.Single(response.Messages); + var contents = response.Messages[0].Contents; + + if (position == "at-start") + { + Assert.Equal(2, contents.Count); + Assert.IsType(contents[0]); + var otherResult = Assert.IsType(contents[1]); + Assert.Equal(otherCallId, otherResult.CallId); + } + else if (position == "at-end") + { + Assert.Equal(2, contents.Count); + var otherResult = Assert.IsType(contents[0]); + Assert.Equal(otherCallId, otherResult.CallId); + Assert.IsType(contents[1]); + } + else + { + Assert.Equal(3, contents.Count); + var beforeResult = Assert.IsType(contents[0]); + Assert.Equal(otherCallId, beforeResult.CallId); + Assert.IsType(contents[1]); + var usageContent = Assert.IsType(contents[2]); + Assert.Equal(5, usageContent.Details.InputTokenCount); + } + } + + [Theory] + [InlineData("at-start")] + [InlineData("in-middle")] + [InlineData("at-end")] + public async Task GetStreamingResponseAsync_FunctionCallContent_SurroundingContentPreservedInOrder(string position) + { + // Regression test for: image generation content duplicating preceding content and dropping following content. + // The image-generation FunctionCallContent should be replaced with ImageGenerationToolCallContent, + // while all other content items retain their original order and cardinality. + var imageCallId = "image-call-id"; + var otherCallId = "other-call-id"; + + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => GetUpdatesAsync() + }; + + async IAsyncEnumerable GetUpdatesAsync() + { + await Task.Yield(); + IList contents = position switch + { + "at-start" => [new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" }), + new FunctionResultContent(otherCallId, "other-result")], + "at-end" => [new FunctionResultContent(otherCallId, "other-result"), + new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" })], + _ => [new FunctionResultContent(otherCallId, "before-result"), + new FunctionCallContent(imageCallId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" }), + new UsageContent(new UsageDetails { InputTokenCount = 5 })], + }; + yield return new ChatResponseUpdate(ChatRole.Assistant, contents); + } + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var updates = new List(); + await foreach (var update in client.GetStreamingResponseAsync([new(ChatRole.User, "test")], chatOptions)) + { + updates.Add(update); + } + + // Assert + Assert.Single(updates); + var contents = updates[0].Contents; + + if (position == "at-start") + { + Assert.Equal(2, contents.Count); + Assert.IsType(contents[0]); + var otherResult = Assert.IsType(contents[1]); + Assert.Equal(otherCallId, otherResult.CallId); + } + else if (position == "at-end") + { + Assert.Equal(2, contents.Count); + var otherResult = Assert.IsType(contents[0]); + Assert.Equal(otherCallId, otherResult.CallId); + Assert.IsType(contents[1]); + } + else + { + Assert.Equal(3, contents.Count); + var beforeResult = Assert.IsType(contents[0]); + Assert.Equal(otherCallId, beforeResult.CallId); + Assert.IsType(contents[1]); + var usageContent = Assert.IsType(contents[2]); + Assert.Equal(5, usageContent.Details.InputTokenCount); + } + } }