diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index c644bd77f21..98cf49fd696 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -157,30 +157,54 @@ void IDisposable.Dispose() } else if (input.Role == ChatRole.Assistant) { - AssistantChatMessage message = new(ToOpenAIChatContent(input.Contents)) - { - ParticipantName = input.AuthorName - }; - + List? contentParts = null; + List? toolCalls = null; + string? refusal = null; foreach (var content in input.Contents) { switch (content) { - case ErrorContent errorContent when errorContent.ErrorCode is nameof(message.Refusal): - message.Refusal = errorContent.Message; + case ErrorContent ec when ec.ErrorCode == nameof(AssistantChatMessage.Refusal): + refusal = ec.Message; break; - case FunctionCallContent callRequest: - message.ToolCalls.Add( - ChatToolCall.CreateFunctionToolCall( - callRequest.CallId, - callRequest.Name, - new(JsonSerializer.SerializeToUtf8Bytes( - callRequest.Arguments, - options.GetTypeInfo(typeof(IDictionary)))))); + case FunctionCallContent fc: + (toolCalls ??= []).Add( + ChatToolCall.CreateFunctionToolCall(fc.CallId, fc.Name, new(JsonSerializer.SerializeToUtf8Bytes( + fc.Arguments, options.GetTypeInfo(typeof(IDictionary)))))); break; + + default: + if (ToChatMessageContentPart(content) is { } part) + { + (contentParts ??= []).Add(part); + } + + break; + } + } + + AssistantChatMessage message; + if (contentParts is not null) + { + message = new(contentParts); + if (toolCalls is not null) + { + foreach (var toolCall in toolCalls) + { + message.ToolCalls.Add(toolCall); + } } } + else + { + message = toolCalls is not null ? + new(toolCalls) : + new(ChatMessageContentPart.CreateTextPart(string.Empty)); + } + + message.ParticipantName = input.AuthorName; + message.Refusal = refusal; yield return message; } @@ -191,38 +215,12 @@ void IDisposable.Dispose() private static List ToOpenAIChatContent(IList contents) { List parts = []; + foreach (var content in contents) { - switch (content) + if (ToChatMessageContentPart(content) is { } part) { - case TextContent textContent: - parts.Add(ChatMessageContentPart.CreateTextPart(textContent.Text)); - break; - - case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): - var audioData = BinaryData.FromBytes(dataContent.Data); - if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3)); - } - else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)); - } - - break; - - case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): - parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf")); - break; + parts.Add(part); } } @@ -234,6 +232,39 @@ private static List ToOpenAIChatContent(IList return parts; } + private static ChatMessageContentPart? ToChatMessageContentPart(AIContent content) + { + switch (content) + { + case TextContent textContent: + return ChatMessageContentPart.CreateTextPart(textContent.Text); + + case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): + var audioData = BinaryData.FromBytes(dataContent.Data); + if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3); + } + else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav); + } + + break; + + case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): + return ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"); + } + + return null; + } + private static ChatImageDetailLevel? GetImageDetail(AIContent content) { if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true) diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index edb6c5dd14c..994fef47517 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -77,7 +77,9 @@ public virtual async Task GetResponseAsync_WithEmptyMessage() var response = await _chatClient.GetResponseAsync( [ + new(ChatRole.System, []), new(ChatRole.User, []), + new(ChatRole.Assistant, []), new(ChatRole.User, "What is 1 + 2? Reply with a single number."), ]); @@ -618,9 +620,11 @@ public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputUnchange var secondResponse = await chatClient.GetResponseAsync([message]); Assert.Equal(response.Text, secondResponse.Text); Assert.Equal(2, functionCallCount); - Assert.Equal(2, llmCallCount!.CallCount); + Assert.Equal(FunctionInvokingChatClientSetsConversationId ? 3 : 2, llmCallCount!.CallCount); } + public virtual bool FunctionInvokingChatClientSetsConversationId => false; + [ConditionalFact] public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputChangedAsync() { diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index bbfad1c571d..2c1d6cdc80e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -9,4 +9,6 @@ public class OpenAIResponseClientIntegrationTests : ChatClientIntegrationTests IntegrationTestHelpers.GetOpenAIClient() ?.GetOpenAIResponseClient(TestRunnerConfiguration.Instance["OpenAI:ChatModel"] ?? "gpt-4o-mini") .AsIChatClient(); + + public override bool FunctionInvokingChatClientSetsConversationId => true; }