From da0b9ec30183d1104d7f5e307d3d8a9d28fd9dfc Mon Sep 17 00:00:00 2001 From: Waldek Mastykarz Date: Wed, 14 May 2025 15:54:18 +0200 Subject: [PATCH] Adds support for OAI Content Parts. Closes #1174 --- .../ILanguageModelChatCompletionMessage.cs | 2 +- .../LanguageModel/OllamaModels.cs | 6 +- .../OpenAIContentPartJsonConverter.cs | 75 +++++++++++++++++ .../OpenAILanguageModelClient.cs | 6 +- .../LanguageModel/OpenAIModels.cs | 80 ++++++++++++++++--- .../Mocks/OpenAIMockResponsePlugin.cs | 7 ++ 6 files changed, 157 insertions(+), 19 deletions(-) create mode 100644 dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs diff --git a/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs b/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs index 518c8455..8ded3e06 100644 --- a/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs +++ b/dev-proxy-abstractions/LanguageModel/ILanguageModelChatCompletionMessage.cs @@ -6,6 +6,6 @@ namespace DevProxy.Abstractions.LanguageModel; public interface ILanguageModelChatCompletionMessage { - string Content { get; set; } + object Content { get; set; } string Role { get; set; } } \ No newline at end of file diff --git a/dev-proxy-abstractions/LanguageModel/OllamaModels.cs b/dev-proxy-abstractions/LanguageModel/OllamaModels.cs index 15cdfa81..392c17f3 100644 --- a/dev-proxy-abstractions/LanguageModel/OllamaModels.cs +++ b/dev-proxy-abstractions/LanguageModel/OllamaModels.cs @@ -89,7 +89,7 @@ public class OllamaLanguageModelChatCompletionResponse : OllamaResponse public OllamaLanguageModelChatCompletionMessage Message { get; set; } = new(); public override string? Response { - get => Message.Content; + get => Message.Content.ToString(); set { if (value is null) @@ -118,7 +118,7 @@ public override OpenAIResponse ConvertToOpenAIResponse() Index = 0, Message = new() { - Content = Message.Content, + Content = Message.Content.ToString() ?? string.Empty, Role = Message.Role } }], @@ -152,7 +152,7 @@ public override OpenAIResponse ConvertToOpenAIResponse() public class OllamaLanguageModelChatCompletionMessage : ILanguageModelChatCompletionMessage { - public string Content { get; set; } = string.Empty; + public object Content { get; set; } = string.Empty; public string Role { get; set; } = string.Empty; public override bool Equals(object? obj) diff --git a/dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs b/dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs new file mode 100644 index 00000000..31106a50 --- /dev/null +++ b/dev-proxy-abstractions/LanguageModel/OpenAIContentPartJsonConverter.cs @@ -0,0 +1,75 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DevProxy.Abstractions.LanguageModel; + +public class OpenAIContentPartJsonConverter : JsonConverter +{ + public override bool CanConvert(Type typeToConvert) + { + return typeToConvert == typeof(object); + } + + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String) + { + return reader.GetString(); + } + else if (reader.TokenType == JsonTokenType.StartArray) + { + var contentParts = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + { + break; + } + + if (reader.TokenType == JsonTokenType.StartObject) + { + using JsonDocument doc = JsonDocument.ParseValue(ref reader); + + var root = doc.RootElement; + if (root.TryGetProperty("type", out var typeProp)) + { + var contentType = typeProp.GetString() switch + { + "text" => typeof(OpenAITextContentPart), + "image" => typeof(OpenAIImageContentPart), + "audio" => typeof(OpenAIAudioContentPart), + "file" => typeof(OpenAIFileContentPart), + _ => null + }; + if (contentType is not null) + { + var contentPart = JsonSerializer.Deserialize(doc.RootElement.GetRawText(), contentType, options); + if (contentPart is not null) + { + contentParts.Add(contentPart); + } + } + } + } + } + return contentParts.ToArray(); + } + return null; + } + + public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerOptions options) + { + if (value is string str) + { + writer.WriteStringValue(str); + } + else + { + JsonSerializer.Serialize(writer, value, options); + } + } +} \ No newline at end of file diff --git a/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs b/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs index 4cc12e6c..40ba7bd2 100644 --- a/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs +++ b/dev-proxy-abstractions/LanguageModel/OpenAILanguageModelClient.cs @@ -2,9 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using Microsoft.Extensions.Logging; using System.Diagnostics; using System.Net.Http.Json; -using Microsoft.Extensions.Logging; namespace DevProxy.Abstractions.LanguageModel; @@ -170,7 +170,9 @@ private async Task IsEnabledInternalAsync() Temperature = options?.Temperature }; - var response = await _httpClient.PostAsJsonAsync(url, payload); + // var payloadString = JsonSerializer.Serialize(payload, ProxyUtils.JsonSerializerOptions); + + var response = await _httpClient.PostAsJsonAsync(url, payload, ProxyUtils.JsonSerializerOptions); _logger.LogDebug("Response: {response}", response.StatusCode); if (!response.IsSuccessStatusCode) diff --git a/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs b/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs index 2960b3cf..088f1b13 100644 --- a/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs +++ b/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs @@ -38,7 +38,7 @@ public class OpenAIError public string? Message { get; set; } } -public abstract class OpenAIResponse: ILanguageModelCompletionResponse +public abstract class OpenAIResponse : ILanguageModelCompletionResponse { public long Created { get; set; } public OpenAIError? Error { get; set; } @@ -106,9 +106,63 @@ public class OpenAICompletionResponseChoice : OpenAIResponseChoice public string Text { get; set; } = string.Empty; } -public class OpenAIChatCompletionMessage: ILanguageModelChatCompletionMessage +#region content parts + +public abstract class OpenAIContentPart { - public string Content { get; set; } = string.Empty; + public string? Type { get; set; } +} + +public class OpenAITextContentPart : OpenAIContentPart +{ + public string? Text { get; set; } +} + +public class OpenAIImageContentPartUrl +{ + public string? Detail { get; set; } = "auto"; + public string? Url { get; set; } +} + +public class OpenAIImageContentPart : OpenAIContentPart +{ + [JsonPropertyName("image_url")] + public OpenAIImageContentPartUrl? Url { get; set; } +} + +public class OpenAIAudioContentPartInputAudio +{ + public string? Data { get; set; } + public string? Format { get; set; } +} + +public class OpenAIAudioContentPart : OpenAIContentPart +{ + [JsonPropertyName("input_audio")] + public OpenAIAudioContentPartInputAudio? InputAudio { get; set; } +} + +public class OpenAIFileContentPartFile +{ + [JsonPropertyName("file_data")] + public string? Data { get; set; } + [JsonPropertyName("file_id")] + public string? Id { get; set; } + [JsonPropertyName("filename")] + public string? Name { get; set; } +} + +public class OpenAIFileContentPart : OpenAIContentPart +{ + public OpenAIFileContentPartFile? File { get; set; } +} + +#endregion + +public class OpenAIChatCompletionMessage : ILanguageModelChatCompletionMessage +{ + [JsonConverter(typeof(OpenAIContentPartJsonConverter))] + public object Content { get; set; } = string.Empty; public string Role { get; set; } = string.Empty; public override bool Equals(object? obj) @@ -204,13 +258,13 @@ public class OpenAIFineTuneRequest : OpenAIRequest } public class OpenAIFineTuneResponse : OpenAIResponse -{ +{ [JsonPropertyName("fine_tuned_model")] - public string? FineTunedModel { get; set; } - public string Status { get; set; } = string.Empty; - public string? Organization { get; set; } - public long CreatedAt { get; set; } - public long UpdatedAt { get; set; } + public string? FineTunedModel { get; set; } + public string Status { get; set; } = string.Empty; + public string? Organization { get; set; } + public long CreatedAt { get; set; } + public long UpdatedAt { get; set; } [JsonPropertyName("training_file")] public string TrainingFile { get; set; } = string.Empty; [JsonPropertyName("validation_file")] @@ -233,16 +287,16 @@ public class OpenAIImageRequest : OpenAIRequest } public class OpenAIImageResponse : OpenAIResponse -{ - public OpenAIImageData[]? Data { get; set; } +{ + public OpenAIImageData[]? Data { get; set; } public override string? Response => null; // Image responses don't have a text response } public class OpenAIImageData { - public string? Url { get; set; } + public string? Url { get; set; } [JsonPropertyName("b64_json")] - public string? Base64Json { get; set; } + public string? Base64Json { get; set; } [JsonPropertyName("revised_prompt")] public string? RevisedPrompt { get; set; } } diff --git a/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs b/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs index e13bd8e2..fd15e4c9 100644 --- a/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs +++ b/dev-proxy-plugins/Mocks/OpenAIMockResponsePlugin.cs @@ -34,6 +34,13 @@ public override async Task RegisterAsync() private async Task OnRequestAsync(object sender, ProxyRequestArgs e) { + if (UrlsToWatch is null || + !e.HasRequestUrlMatch(UrlsToWatch)) + { + Logger.LogRequest("URL not matched", MessageType.Skipped, new LoggingContext(e.Session)); + return; + } + var request = e.Session.HttpClient.Request; if (request.Method is null || !request.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) ||