diff --git a/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs b/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs index 440d5470..2960b3cf 100644 --- a/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs +++ b/dev-proxy-abstractions/LanguageModel/OpenAIModels.cs @@ -143,3 +143,106 @@ public class OpenAIChatCompletionResponseChoiceMessage public string Content { get; set; } = string.Empty; public string Role { get; set; } = string.Empty; } + +public class OpenAIAudioRequest : OpenAIRequest +{ + public string File { get; set; } = string.Empty; + [JsonPropertyName("response_format")] + public string? ResponseFormat { get; set; } + public string? Prompt { get; set; } + public string? Language { get; set; } +} + +public class OpenAIAudioSpeechRequest : OpenAIRequest +{ + public string Input { get; set; } = string.Empty; + public string Voice { get; set; } = string.Empty; + [JsonPropertyName("response_format")] + public string? ResponseFormat { get; set; } + public double? Speed { get; set; } +} + +public class OpenAIAudioTranscriptionResponse : OpenAIResponse +{ + public string Text { get; set; } = string.Empty; + public override string? Response => Text; +} + +public class OpenAIEmbeddingRequest : OpenAIRequest +{ + public string? Input { get; set; } + [JsonPropertyName("encoding_format")] + public string? EncodingFormat { get; set; } + public int? Dimensions { get; set; } +} + +public class OpenAIEmbeddingResponse : OpenAIResponse +{ + public OpenAIEmbeddingData[]? Data { get; set; } + public override string? Response => null; // Embeddings don't have a text response +} + +public class OpenAIEmbeddingData +{ + public float[]? Embedding { get; set; } + public int Index { get; set; } + public string? Object { get; set; } +} + +public class OpenAIFineTuneRequest : OpenAIRequest +{ + [JsonPropertyName("training_file")] + public string TrainingFile { get; set; } = string.Empty; + [JsonPropertyName("validation_file")] + public string? ValidationFile { get; set; } + public int? Epochs { get; set; } + [JsonPropertyName("batch_size")] + public int? BatchSize { get; set; } + [JsonPropertyName("learning_rate_multiplier")] + public double? LearningRateMultiplier { get; set; } + public string? Suffix { get; set; } +} + +public class OpenAIFineTuneResponse : OpenAIResponse +{ + [JsonPropertyName("fine_tuned_model")] + public string? FineTunedModel { get; set; } + public string Status { get; set; } = string.Empty; + public string? Organization { get; set; } + public long CreatedAt { get; set; } + public long UpdatedAt { get; set; } + [JsonPropertyName("training_file")] + public string TrainingFile { get; set; } = string.Empty; + [JsonPropertyName("validation_file")] + public string? ValidationFile { get; set; } + [JsonPropertyName("result_files")] + public object[]? ResultFiles { get; set; } + public override string? Response => FineTunedModel; +} + +public class OpenAIImageRequest : OpenAIRequest +{ + public string Prompt { get; set; } = string.Empty; + public int? N { get; set; } + public string? Size { get; set; } + [JsonPropertyName("response_format")] + public string? ResponseFormat { get; set; } + public string? User { get; set; } + public string? Quality { get; set; } + public string? Style { get; set; } +} + +public class OpenAIImageResponse : OpenAIResponse +{ + public OpenAIImageData[]? Data { get; set; } + public override string? Response => null; // Image responses don't have a text response +} + +public class OpenAIImageData +{ + public string? Url { get; set; } + [JsonPropertyName("b64_json")] + public string? Base64Json { get; set; } + [JsonPropertyName("revised_prompt")] + public string? RevisedPrompt { get; set; } +} diff --git a/dev-proxy-abstractions/LanguageModel/PricesData.cs b/dev-proxy-abstractions/LanguageModel/PricesData.cs new file mode 100644 index 00000000..83681f4b --- /dev/null +++ b/dev-proxy-abstractions/LanguageModel/PricesData.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics; + +namespace DevProxy.Abstractions.LanguageModel; + +public class PricesData: Dictionary +{ + public bool TryGetModelPrices(string modelName, out ModelPrices? prices) + { + prices = new ModelPrices(); + + if (string.IsNullOrEmpty(modelName)) + { + return false; + } + + // Try exact match first + if (TryGetValue(modelName, out prices)) + { + return true; + } + + // Try to find a matching prefix + // This handles cases like "gpt-4-turbo-2024-04-09" matching with "gpt-4" + var matchingModel = Keys + .Where(k => modelName.StartsWith(k, StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(k => k.Length) + .FirstOrDefault(); + + if (matchingModel != null && TryGetValue(matchingModel, out prices)) + { + return true; + } + + return false; + } + + public (double Input, double Output) CalculateCost(string modelName, long inputTokens, long outputTokens) + { + if (!TryGetModelPrices(modelName, out var prices)) + { + return (0, 0); + } + + Debug.Assert(prices != null, "Prices data should not be null here."); + + // Prices in the data are per 1M tokens + var inputCost = prices.Input * (inputTokens / 1_000_000.0); + var outputCost = prices.Output * (outputTokens / 1_000_000.0); + + return (inputCost, outputCost); + } +} + +public class ModelPrices +{ + public double Input { get; set; } + public double Output { get; set; } +} diff --git a/dev-proxy-abstractions/OpenTelemetry/SemanticConvention.cs b/dev-proxy-abstractions/OpenTelemetry/SemanticConvention.cs new file mode 100644 index 00000000..3c0ad1f0 --- /dev/null +++ b/dev-proxy-abstractions/OpenTelemetry/SemanticConvention.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace DevProxy.Abstractions.OpenTelemetry; + +public static class SemanticConvention +{ + // GenAI General + public const string GEN_AI_ENDPOINT = "gen_ai.endpoint"; + public const string GEN_AI_SYSTEM = "gen_ai.system"; + public const string GEN_AI_ENVIRONMENT = "gen_ai.environment"; + public const string GEN_AI_APPLICATION_NAME = "gen_ai.application_name"; + public const string GEN_AI_OPERATION = "gen_ai.type"; + public const string GEN_AI_OPERATION_NAME = "gen_ai.operation.name"; + public const string GEN_AI_HUB_OWNER = "gen_ai.hub.owner"; + public const string GEN_AI_HUB_REPO = "gen_ai.hub.repo"; + public const string GEN_AI_RETRIEVAL_SOURCE = "gen_ai.retrieval.source"; + public const string GEN_AI_REQUESTS = "gen_ai.total.requests"; + + // GenAI Request + public const string GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; + public const string GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; + public const string GEN_AI_REQUEST_TOP_P = "gen_ai.request.top_p"; + public const string GEN_AI_REQUEST_TOP_K = "gen_ai.request.top_k"; + public const string GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; + public const string GEN_AI_REQUEST_IS_STREAM = "gen_ai.request.is_stream"; + public const string GEN_AI_REQUEST_USER = "gen_ai.request.user"; + public const string GEN_AI_REQUEST_SEED = "gen_ai.request.seed"; + public const string GEN_AI_REQUEST_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty"; + public const string GEN_AI_REQUEST_PRESENCE_PENALTY = "gen_ai.request.presence_penalty"; + public const string GEN_AI_REQUEST_ENCODING_FORMATS = "gen_ai.request.embedding_format"; + public const string GEN_AI_REQUEST_EMBEDDING_DIMENSION = "gen_ai.request.embedding_dimension"; + public const string GEN_AI_REQUEST_TOOL_CHOICE = "gen_ai.request.tool_choice"; + public const string GEN_AI_REQUEST_AUDIO_VOICE = "gen_ai.request.audio_voice"; + public const string GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT = "gen_ai.request.audio_response_format"; + public const string GEN_AI_REQUEST_AUDIO_SPEED = "gen_ai.request.audio_speed"; + public const string GEN_AI_REQUEST_FINETUNE_STATUS = "gen_ai.request.fine_tune_status"; + public const string GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX = "gen_ai.request.fine_tune_model_suffix"; + public const string GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS = "gen_ai.request.fine_tune_n_epochs"; + public const string GEN_AI_REQUEST_FINETUNE_MODEL_LRM = "gen_ai.request.learning_rate_multiplier"; + public const string GEN_AI_REQUEST_FINETUNE_BATCH_SIZE = "gen_ai.request.fine_tune_batch_size"; + public const string GEN_AI_REQUEST_VALIDATION_FILE = "gen_ai.request.validation_file"; + public const string GEN_AI_REQUEST_TRAINING_FILE = "gen_ai.request.training_file"; + public const string GEN_AI_REQUEST_IMAGE_SIZE = "gen_ai.request.image_size"; + public const string GEN_AI_REQUEST_IMAGE_QUALITY = "gen_ai.request.image_quality"; + public const string GEN_AI_REQUEST_IMAGE_STYLE = "gen_ai.request.image_style"; + + // GenAI Usage + public const string GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public const string GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + // OpenLIT + public const string GEN_AI_USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"; + public const string GEN_AI_USAGE_COST = "gen_ai.usage.cost"; + public const string GEN_AI_USAGE_TOTAL_COST = "gen_ai.usage.total_cost"; + + // GenAI Response + public const string GEN_AI_RESPONSE_ID = "gen_ai.response.id"; + public const string GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; + public const string GEN_AI_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reason"; + public const string GEN_AI_RESPONSE_IMAGE = "gen_ai.response.image"; + public const string GEN_AI_RESPONSE_IMAGE_SIZE = "gen_ai.request.image_size"; + public const string GEN_AI_RESPONSE_IMAGE_QUALITY = "gen_ai.request.image_quality"; + public const string GEN_AI_RESPONSE_IMAGE_STYLE = "gen_ai.request.image_style"; + + // GenAI Content + public const string GEN_AI_CONTENT_PROMPT = "gen_ai.content.prompt"; + public const string GEN_AI_CONTENT_COMPLETION = "gen_ai.completion"; + public const string GEN_AI_CONTENT_REVISED_PROMPT = "gen_ai.content.revised_prompt"; + + // Operation Types + public const string GEN_AI_OPERATION_TYPE_CHAT = "chat"; + public const string GEN_AI_OPERATION_TYPE_EMBEDDING = "embedding"; + public const string GEN_AI_OPERATION_TYPE_IMAGE = "image"; + public const string GEN_AI_OPERATION_TYPE_AUDIO = "audio"; + public const string GEN_AI_OPERATION_TYPE_FINETUNING = "fine_tuning"; + public const string GEN_AI_OPERATION_TYPE_VECTORDB = "vectordb"; + public const string GEN_AI_OPERATION_TYPE_FRAMEWORK = "framework"; + + // Metrics + public const string GEN_AI_METRIC_CLIENT_TOKEN_USAGE = "gen_ai.client.token.usage"; + public const string GEN_AI_TOKEN_TYPE = "gen_ai.token.type"; + public const string GEN_AI_TOKEN_TYPE_INPUT = "input"; + public const string GEN_AI_TOKEN_TYPE_OUTPUT = "output"; + public const string GEN_AI_METRIC_CLIENT_OPERATION_DURATION = "gen_ai.client.operation.duration"; +} \ No newline at end of file diff --git a/dev-proxy-plugins/Inspection/LanguageModelPricingLoader.cs b/dev-proxy-plugins/Inspection/LanguageModelPricingLoader.cs new file mode 100644 index 00000000..55a73eb2 --- /dev/null +++ b/dev-proxy-plugins/Inspection/LanguageModelPricingLoader.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions; +using DevProxy.Abstractions.LanguageModel; +using Microsoft.Extensions.Logging; +using System.Text.Json; + +namespace DevProxy.Plugins.Inspection; + +internal class LanguageModelPricesLoader(ILogger logger, LanguageModelPricesPluginConfiguration configuration, bool validateSchemas) : BaseLoader(logger, validateSchemas) +{ + private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + private readonly LanguageModelPricesPluginConfiguration _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + protected override string FilePath => Path.Combine(Directory.GetCurrentDirectory(), _configuration.PricesFile ?? ""); + + protected override void LoadData(string fileContents) + { + try + { + // we need to deserialize manually because standard deserialization + // doesn't support nested dictionaries + using JsonDocument document = JsonDocument.Parse(fileContents); + + if (document.RootElement.TryGetProperty("prices", out JsonElement pricesElement)) + { + var pricesData = new PricesData(); + + foreach (JsonProperty modelProperty in pricesElement.EnumerateObject()) + { + var modelName = modelProperty.Name; + if (modelProperty.Value.TryGetProperty("input", out JsonElement inputElement) && + modelProperty.Value.TryGetProperty("output", out JsonElement outputElement)) + { + pricesData[modelName] = new() + { + Input = inputElement.GetDouble(), + Output = outputElement.GetDouble() + }; + } + } + + _configuration.Prices = pricesData; + _logger.LogInformation("Language model prices data loaded from {PricesFile}", _configuration.PricesFile); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "An error has occurred while reading {PricesFile}:", _configuration.PricesFile); + } + } +} diff --git a/dev-proxy-plugins/Inspection/OpenAITelemetryPlugin.cs b/dev-proxy-plugins/Inspection/OpenAITelemetryPlugin.cs new file mode 100644 index 00000000..41ad52a9 --- /dev/null +++ b/dev-proxy-plugins/Inspection/OpenAITelemetryPlugin.cs @@ -0,0 +1,934 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions; +using DevProxy.Abstractions.LanguageModel; +using DevProxy.Abstractions.OpenTelemetry; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Text.Json; + +namespace DevProxy.Plugins.Inspection; + +public class LanguageModelPricesPluginConfiguration +{ + public PricesData? Prices; + public string? PricesFile { get; set; } +} + +public class OpenAITelemetryPluginConfiguration : LanguageModelPricesPluginConfiguration +{ + public string Application { get; set; } = "default"; + public string Currency { get; set; } = "USD"; + public string Environment { get; set; } = "development"; + public string ExporterEndpoint { get; set; } = "http://localhost:4318"; + public bool IncludePrompt { get; set; } = true; + public bool IncludeCompletion { get; set; } = true; + public bool IncludeCosts { get; set; } = false; +} + +public class OpenAITelemetryPlugin(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet urlsToWatch, IConfigurationSection? configSection = null) : BaseProxyPlugin(pluginEvents, context, logger, urlsToWatch, configSection), IDisposable +{ + public override string Name => nameof(OpenAITelemetryPlugin); + private readonly OpenAITelemetryPluginConfiguration _configuration = new(); + private LanguageModelPricesLoader? _loader = null; + + private const string ActivitySourceName = "DevProxy.OpenAI"; + private const string OpenAISystem = "openai"; + private readonly ActivitySource _activitySource = new(ActivitySourceName); + private readonly static Meter _meter = new(ActivitySourceName); + private TracerProvider? _tracerProvider; + private MeterProvider? _meterProvider; + + private static Histogram? _tokenUsageMetric; + private static Histogram? _requestCostMetric; + private static Counter? _totalCostMetric; + + public override async Task RegisterAsync() + { + await base.RegisterAsync(); + + ConfigSection?.Bind(_configuration); + + if (_configuration.IncludeCosts) + { + _configuration.PricesFile = Path.GetFullPath(ProxyUtils.ReplacePathTokens(_configuration.PricesFile), Path.GetDirectoryName(Context.Configuration.ConfigFile ?? string.Empty) ?? string.Empty); + _loader = new LanguageModelPricesLoader(Logger, _configuration, Context.Configuration.ValidateSchemas); + _loader.InitFileWatcher(); + } + + InitializeOpenTelemetryExporter(); + + PluginEvents.BeforeRequest += OnRequestAsync; + PluginEvents.BeforeResponse += OnResponseAsync; + } + + private void InitializeOpenTelemetryExporter() + { + Logger.LogTrace("InitializeOpenTelemetryExporter() called"); + + try + { + var resourceBuilder = ResourceBuilder + .CreateDefault() + .AddService(serviceName: "DevProxy.OpenAI", serviceVersion: ProxyUtils.ProductVersion); + + _tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource(ActivitySourceName) + .AddOtlpExporter(options => + { + // We use protobuf to allow intercepting Dev Proxy's own LLM traffic + options.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; + options.Endpoint = new Uri(_configuration.ExporterEndpoint + "/v1/traces"); + }) + .Build(); + + _meterProvider = Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddMeter(ActivitySourceName) + .AddView(SemanticConvention.GEN_AI_METRIC_CLIENT_TOKEN_USAGE, new ExplicitBucketHistogramConfiguration + { + Boundaries = [1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864] + }) + .AddView(SemanticConvention.GEN_AI_USAGE_COST, new ExplicitBucketHistogramConfiguration + { + Boundaries = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100] + }) + .AddView(SemanticConvention.GEN_AI_USAGE_TOTAL_COST, new MetricStreamConfiguration()) + .AddOtlpExporter(options => + { + // We use protobuf to allow intercepting Dev Proxy's own LLM traffic + options.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; + options.Endpoint = new Uri(_configuration.ExporterEndpoint + "/v1/metrics"); + }) + .Build(); + + _tokenUsageMetric = _meter.CreateHistogram( + SemanticConvention.GEN_AI_METRIC_CLIENT_TOKEN_USAGE, + "tokens", + "Number of tokens processed"); + _requestCostMetric = _meter.CreateHistogram( + SemanticConvention.GEN_AI_USAGE_COST, + "cost", + $"Estimated cost per request in {_configuration.Currency}"); + _totalCostMetric = _meter.CreateCounter( + SemanticConvention.GEN_AI_USAGE_TOTAL_COST, + "cost", + $"Total estimated cost for the session in {_configuration.Currency}"); + } + catch (Exception ex) + { + Logger.LogError(ex, "Failed to initialize OpenTelemetry exporter"); + } + + Logger.LogTrace("InitializeOpenTelemetryExporter() finished"); + } + + private async Task OnRequestAsync(object sender, ProxyRequestArgs e) + { + Logger.LogTrace("OnRequestAsync() called"); + + var request = e.Session.HttpClient.Request; + if (request.Method is null || + !request.Method.Equals("POST", StringComparison.OrdinalIgnoreCase) || + !request.HasBody) + { + Logger.LogRequest("Request is not a POST request with a body", MessageType.Skipped, new LoggingContext(e.Session)); + return; + } + + if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest) || openAiRequest is null) + { + Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session)); + return; + } + + // store for use in response + e.SessionData["OpenAIRequest"] = openAiRequest; + + var activity = _activitySource.StartActivity( + $"openai.{GetOperationName(openAiRequest)}", + ActivityKind.Client); + + if (activity is null) + { + Logger.LogWarning("Failed to start OpenTelemetry activity for OpenAI request"); + return; + } + + // add generic request tags + activity.SetTag("http.method", request.Method); + activity.SetTag("http.url", request.RequestUri.ToString()); + activity.SetTag("http.scheme", request.RequestUri.Scheme); + activity.SetTag("http.host", request.RequestUri.Host); + activity.SetTag("http.target", request.RequestUri.PathAndQuery); + activity.SetTag(SemanticConvention.GEN_AI_SYSTEM, OpenAISystem); + activity.SetTag(SemanticConvention.GEN_AI_ENVIRONMENT, _configuration.Environment); + activity.SetTag(SemanticConvention.GEN_AI_APPLICATION_NAME, _configuration.Application); + + AddCommonRequestTags(activity, openAiRequest); + AddRequestTypeSpecificTags(activity, openAiRequest); + + // store for use in response + e.SessionData["OpenAIActivity"] = activity; + + Logger.LogTrace("OnRequestAsync() finished"); + + await Task.CompletedTask; + } + + private async Task OnResponseAsync(object sender, ProxyResponseArgs e) + { + Logger.LogTrace("OnResponseAsync() called"); + + if (!e.SessionData.TryGetValue("OpenAIActivity", out var activityObj) || + activityObj is not Activity activity) + { + return; + } + + try + { + var response = e.Session.HttpClient.Response; + + activity.SetTag("http.status_code", response.StatusCode); + + switch (response.StatusCode) + { + case int code when code >= 200 && code < 300: + ProcessSuccessResponse(activity, e); + break; + case int code when code >= 400: + ProcessErrorResponse(activity, e); + break; + } + } + finally + { + activity.Stop(); + + // Clean up session data + e.SessionData.Remove("OpenAIActivity"); + e.SessionData.Remove("OpenAIRequest"); + + Logger.LogRequest("OpenTelemetry information emitted", MessageType.Processed, new LoggingContext(e.Session)); + } + + await Task.CompletedTask; + } + + private void ProcessErrorResponse(Activity activity, ProxyResponseArgs e) + { + Logger.LogTrace("ProcessErrorResponse() called"); + + var response = e.Session.HttpClient.Response; + + activity.SetTag("error", true); + activity.SetTag("error.type", "http"); + activity.SetTag("error.message", $"HTTP {response.StatusCode}"); + + if (response.HasBody) + { + try + { + var errorObj = JsonSerializer.Deserialize(response.BodyString); + if (errorObj.TryGetProperty("error", out var error)) + { + if (error.TryGetProperty("message", out var message)) + activity.SetTag("error.details", message.GetString()); + } + } + catch (JsonException) + { + // Ignore JSON parsing errors in error responses + } + } + + Logger.LogTrace("ProcessErrorResponse() finished"); + } + + private void ProcessSuccessResponse(Activity activity, ProxyResponseArgs e) + { + Logger.LogTrace("ProcessSuccessResponse() called"); + + var response = e.Session.HttpClient.Response; + + if (!response.HasBody || string.IsNullOrEmpty(response.BodyString)) + { + Logger.LogDebug("Response body is empty or null"); + return; + } + + if (!e.SessionData.TryGetValue("OpenAIRequest", out var requestObj) || + requestObj is not OpenAIRequest openAiRequest) + { + Logger.LogDebug("OpenAI request not found in session data"); + return; + } + + AddResponseTypeSpecificTags(activity, openAiRequest, response.BodyString); + + Logger.LogTrace("ProcessSuccessResponse() finished"); + } + + private void AddResponseTypeSpecificTags(Activity activity, OpenAIRequest openAiRequest, string responseBody) + { + Logger.LogTrace("AddResponseTypeSpecificTags() called"); + + try + { + switch (openAiRequest) + { + case OpenAIChatCompletionRequest: + AddChatCompletionResponseTags(activity, openAiRequest, responseBody); + break; + case OpenAICompletionRequest: + AddCompletionResponseTags(activity, openAiRequest, responseBody); + break; + case OpenAIEmbeddingRequest: + AddEmbeddingResponseTags(activity, openAiRequest, responseBody); + break; + case OpenAIImageRequest: + AddImageResponseTags(activity, openAiRequest, responseBody); + break; + case OpenAIAudioRequest: + AddAudioResponseTags(activity, openAiRequest, responseBody); + break; + case OpenAIFineTuneRequest: + AddFineTuneResponseTags(activity, openAiRequest, responseBody); + break; + } + } + catch (JsonException ex) + { + Logger.LogError(ex, "Failed to deserialize OpenAI response"); + activity.SetTag("error", ex.Message); + } + } + + private void AddFineTuneResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddFineTuneResponseTags() called"); + + var fineTuneResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (fineTuneResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, fineTuneResponse); + + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FINETUNE_STATUS, fineTuneResponse.Status); + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, fineTuneResponse.Id); + + if (!string.IsNullOrEmpty(fineTuneResponse.FineTunedModel)) + { + activity.SetTag("ai.response.fine_tuned_model", fineTuneResponse.FineTunedModel); + } + + Logger.LogTrace("AddFineTuneResponseTags() finished"); + } + + private void AddAudioResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddAudioResponseTags() called"); + + var audioResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (audioResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, audioResponse); + + // Record the transcription text if configured + if (_configuration.IncludeCompletion && !string.IsNullOrEmpty(audioResponse.Text)) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_COMPLETION, audioResponse.Text); + } + + Logger.LogTrace("AddAudioResponseTags() finished"); + } + + private void AddImageResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddImageResponseTags() called"); + + var imageResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (imageResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, imageResponse); + + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, imageResponse.Id); + + if (imageResponse.Data != null) + { + activity.SetTag("ai.response.image.count", imageResponse.Data.Length); + + if (_configuration.IncludeCompletion && + imageResponse.Data.Length > 0 && + !string.IsNullOrEmpty(imageResponse.Data[0]?.RevisedPrompt)) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_REVISED_PROMPT, + imageResponse.Data[0].RevisedPrompt); + } + } + + Logger.LogTrace("AddImageResponseTags() finished"); + } + + private void AddEmbeddingResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddEmbeddingResponseTags() called"); + + var embeddingResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (embeddingResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, embeddingResponse); + + // Embedding response doesn't have a "completion" but we can record some metadata + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, embeddingResponse.Id); + if (embeddingResponse.Data is not null) + { + activity.SetTag("ai.embedding.count", embeddingResponse.Data.Length); + + // If there's only one embedding, record the dimensions + if (embeddingResponse.Data.Length == 1 && + embeddingResponse.Data[0]?.Embedding is not null) + { + activity.SetTag("ai.embedding.dimensions", embeddingResponse.Data[0]?.Embedding?.Length ?? 0); + } + } + + Logger.LogTrace("AddEmbeddingResponseTags() finished"); + } + + private void AddChatCompletionResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddChatCompletionResponseTags() called"); + + var chatResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (chatResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, chatResponse); + + if (chatResponse.Choices?.Length > 0 && chatResponse.Choices[0] != null && chatResponse.Choices[0].Message != null) + { + if (_configuration.IncludeCompletion) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_COMPLETION, chatResponse.Choices[0].Message.Content); + } + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, chatResponse.Choices[0].FinishReason); + } + + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, chatResponse.Id); + + Logger.LogTrace("AddChatCompletionResponseTags() finished"); + } + + private void AddCompletionResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddCompletionResponseTags() called"); + + var completionResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (completionResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, completionResponse); + + if (completionResponse.Choices?.Length > 0 && completionResponse.Choices[0] is not null) + { + if (_configuration.IncludeCompletion) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_COMPLETION, completionResponse.Choices[0].Text); + } + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON, completionResponse.Choices[0].FinishReason); + } + + activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, completionResponse.Id); + + Logger.LogTrace("AddCompletionResponseTags() finished"); + } + + private void AddRequestTypeSpecificTags(Activity activity, OpenAIRequest openAiRequest) + { + switch (openAiRequest) + { + case OpenAIChatCompletionRequest chatRequest: + AddChatCompletionRequestTags(activity, chatRequest); + break; + case OpenAICompletionRequest completionRequest: + AddCompletionRequestTags(activity, completionRequest); + break; + case OpenAIEmbeddingRequest embeddingRequest: + AddEmbeddingRequestTags(activity, embeddingRequest); + break; + case OpenAIImageRequest imageRequest: + AddImageRequestTags(activity, imageRequest); + break; + case OpenAIAudioRequest audioRequest: + AddAudioRequestTags(activity, audioRequest); + break; + case OpenAIAudioSpeechRequest speechRequest: + AddAudioSpeechRequestTags(activity, speechRequest); + break; + case OpenAIFineTuneRequest fineTuneRequest: + AddFineTuneRequestTags(activity, fineTuneRequest); + break; + } + } + + private void AddCompletionRequestTags(Activity activity, OpenAICompletionRequest completionRequest) + { + Logger.LogTrace("AddCompletionRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_CONTENT_COMPLETION); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_CONTENT_COMPLETION); + + if (_configuration.IncludePrompt) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, completionRequest.Prompt); + } + + Logger.LogTrace("AddCompletionRequestTags() finished"); + } + + private void AddChatCompletionRequestTags(Activity activity, OpenAIChatCompletionRequest chatRequest) + { + Logger.LogTrace("AddChatCompletionRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT); + + if (_configuration.IncludePrompt) + { + // Format messages to a more readable form for the span + var formattedMessages = chatRequest.Messages + .Select(m => $"{m.Role}: {m.Content}") + .ToArray(); + + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, string.Join("\n", formattedMessages)); + } + + Logger.LogTrace("AddChatCompletionRequestTags() finished"); + } + + private void AddEmbeddingRequestTags(Activity activity, OpenAIEmbeddingRequest embeddingRequest) + { + Logger.LogTrace("AddEmbeddingRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_EMBEDDING); + + if (_configuration.IncludePrompt && embeddingRequest.Input is not null) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, embeddingRequest.Input); + } + + if (embeddingRequest.EncodingFormat is not null) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_ENCODING_FORMATS, embeddingRequest.EncodingFormat); + } + + if (embeddingRequest.Dimensions.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_EMBEDDING_DIMENSION, embeddingRequest.Dimensions.Value); + } + + Logger.LogTrace("AddEmbeddingRequestTags() finished"); + } + + private void AddImageRequestTags(Activity activity, OpenAIImageRequest imageRequest) + { + Logger.LogTrace("AddImageRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_IMAGE); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_IMAGE); + + if (_configuration.IncludePrompt && !string.IsNullOrEmpty(imageRequest.Prompt)) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, imageRequest.Prompt); + } + + if (!string.IsNullOrEmpty(imageRequest.Size)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_IMAGE_SIZE, imageRequest.Size); + } + + if (!string.IsNullOrEmpty(imageRequest.Quality)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_IMAGE_QUALITY, imageRequest.Quality); + } + + if (!string.IsNullOrEmpty(imageRequest.Style)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_IMAGE_STYLE, imageRequest.Style); + } + + if (imageRequest.N.HasValue) + { + activity.SetTag("ai.request.image.count", imageRequest.N.Value); + } + + Logger.LogTrace("AddImageRequestTags() finished"); + } + + private void AddAudioRequestTags(Activity activity, OpenAIAudioRequest audioRequest) + { + Logger.LogTrace("AddAudioRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_AUDIO); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_AUDIO); + + if (!string.IsNullOrEmpty(audioRequest.ResponseFormat)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT, audioRequest.ResponseFormat); + } + + if (!string.IsNullOrEmpty(audioRequest.Prompt) && _configuration.IncludePrompt) + { + activity.SetTag("ai.request.audio.prompt", audioRequest.Prompt); + } + + if (!string.IsNullOrEmpty(audioRequest.Language)) + { + activity.SetTag("ai.request.audio.language", audioRequest.Language); + } + + Logger.LogTrace("AddAudioRequestTags() finished"); + } + + private void AddAudioSpeechRequestTags(Activity activity, OpenAIAudioSpeechRequest speechRequest) + { + Logger.LogTrace("AddAudioSpeechRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_AUDIO); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_AUDIO); + + if (_configuration.IncludePrompt && !string.IsNullOrEmpty(speechRequest.Input)) + { + activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, speechRequest.Input); + } + + if (!string.IsNullOrEmpty(speechRequest.Voice)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_AUDIO_VOICE, speechRequest.Voice); + } + + if (!string.IsNullOrEmpty(speechRequest.ResponseFormat)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_AUDIO_RESPONSE_FORMAT, speechRequest.ResponseFormat); + } + + if (speechRequest.Speed.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_AUDIO_SPEED, speechRequest.Speed.Value); + } + + Logger.LogTrace("AddAudioSpeechRequestTags() finished"); + } + + private void AddFineTuneRequestTags(Activity activity, OpenAIFineTuneRequest fineTuneRequest) + { + Logger.LogTrace("AddFineTuneRequestTags() called"); + + // OpenLIT + activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_FINETUNING); + // OpenTelemetry + activity.SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, SemanticConvention.GEN_AI_OPERATION_TYPE_FINETUNING); + + if (!string.IsNullOrEmpty(fineTuneRequest.TrainingFile)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_TRAINING_FILE, fineTuneRequest.TrainingFile); + } + + if (!string.IsNullOrEmpty(fineTuneRequest.ValidationFile)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_VALIDATION_FILE, fineTuneRequest.ValidationFile); + } + + if (fineTuneRequest.BatchSize.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FINETUNE_BATCH_SIZE, fineTuneRequest.BatchSize.Value); + } + + if (fineTuneRequest.LearningRateMultiplier.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FINETUNE_MODEL_LRM, + fineTuneRequest.LearningRateMultiplier.Value); + } + + if (fineTuneRequest.Epochs.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FINETUNE_MODEL_EPOCHS, + fineTuneRequest.Epochs.Value); + } + + if (!string.IsNullOrEmpty(fineTuneRequest.Suffix)) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FINETUNE_MODEL_SUFFIX, fineTuneRequest.Suffix); + } + + Logger.LogTrace("AddFineTuneRequestTags() finished"); + } + + private void AddCommonRequestTags(Activity activity, OpenAIRequest openAiRequest) + { + Logger.LogTrace("AddCommonRequestTags() called"); + + if (openAiRequest.Temperature.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_TEMPERATURE, openAiRequest.Temperature.Value); + } + + if (openAiRequest.MaxTokens.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, openAiRequest.MaxTokens.Value); + } + + if (openAiRequest.TopP.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_TOP_P, openAiRequest.TopP.Value); + } + + if (openAiRequest.PresencePenalty.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_PRESENCE_PENALTY, openAiRequest.PresencePenalty.Value); + } + + if (openAiRequest.FrequencyPenalty.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_FREQUENCY_PENALTY, openAiRequest.FrequencyPenalty.Value); + } + + if (openAiRequest.Stream.HasValue) + { + activity.SetTag(SemanticConvention.GEN_AI_REQUEST_IS_STREAM, openAiRequest.Stream.Value); + } + + Logger.LogTrace("AddCommonRequestTags() finished"); + } + + private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAIResponse response) + { + Logger.LogTrace("RecordUsageMetrics() called"); + + var usage = response.Usage; + if (usage is null) + { + return; + } + + Debug.Assert(_tokenUsageMetric is not null, "Token usage histogram is not initialized"); + _tokenUsageMetric.Record(response.Usage?.PromptTokens ?? 0, + [ + new(SemanticConvention.GEN_AI_OPERATION_NAME, GetOperationName(request)), + new(SemanticConvention.GEN_AI_SYSTEM, OpenAISystem), + new(SemanticConvention.GEN_AI_TOKEN_TYPE, SemanticConvention.GEN_AI_TOKEN_TYPE_INPUT), + new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), + new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) + ]); + _tokenUsageMetric.Record(response.Usage?.CompletionTokens ?? 0, + [ + new(SemanticConvention.GEN_AI_OPERATION_NAME, GetOperationName(request)), + new(SemanticConvention.GEN_AI_SYSTEM, OpenAISystem), + new(SemanticConvention.GEN_AI_TOKEN_TYPE, SemanticConvention.GEN_AI_TOKEN_TYPE_OUTPUT), + new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), + new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) + ]); + + activity.SetTag(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS, usage.PromptTokens); + activity.SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens); + activity.SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens); + + if (!_configuration.IncludeCosts || _configuration.Prices is null) + { + Logger.LogDebug("Cost tracking is disabled or prices data is not available"); + return; + } + + if (string.IsNullOrEmpty(response.Model)) + { + Logger.LogDebug("Response model is empty or null"); + return; + } + + var (inputCost, outputCost) = _configuration.Prices.CalculateCost(response.Model, usage.PromptTokens, usage.CompletionTokens); + + if (inputCost > 0) + { + var totalCost = inputCost + outputCost; + activity.SetTag(SemanticConvention.GEN_AI_USAGE_COST, totalCost); + + Debug.Assert(_requestCostMetric is not null, "Cost histogram is not initialized"); + Debug.Assert(_totalCostMetric is not null, "Total cost counter is not initialized"); + + _requestCostMetric.Record(totalCost, + [ + new(SemanticConvention.GEN_AI_OPERATION_NAME, GetOperationName(request)), + new(SemanticConvention.GEN_AI_SYSTEM, OpenAISystem), + new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), + new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) + ]); + _totalCostMetric.Add(totalCost, + [ + new(SemanticConvention.GEN_AI_OPERATION_NAME, GetOperationName(request)), + new(SemanticConvention.GEN_AI_SYSTEM, OpenAISystem), + new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), + new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) + ]); + } + else + { + Logger.LogDebug("Input cost is zero, skipping cost metrics recording"); + } + + Logger.LogTrace("RecordUsageMetrics() finished"); + } + + private static string GetOperationName(OpenAIRequest request) + { + if (request == null) + { + return "unknown"; + } + + return request switch + { + OpenAIChatCompletionRequest => "chat.completions", + OpenAICompletionRequest => "completions", + OpenAIEmbeddingRequest => "embeddings", + OpenAIImageRequest => "images.generations", + OpenAIAudioRequest => "audio.transcriptions", + OpenAIAudioSpeechRequest => "audio.speech", + OpenAIFineTuneRequest => "fine_tuning.jobs", + _ => "unknown" + }; + } + + private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) + { + Logger.LogTrace("TryGetOpenAIRequest() called"); + + request = null; + + if (string.IsNullOrEmpty(content)) + { + Logger.LogDebug("Request content is empty or null"); + return false; + } + + try + { + Logger.LogDebug("Checking if the request is an OpenAI request..."); + + var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + + // Check for completion request (has "prompt", but not specific to image) + if (rawRequest.TryGetProperty("prompt", out _) && + !rawRequest.TryGetProperty("size", out _) && + !rawRequest.TryGetProperty("n", out _)) + { + Logger.LogDebug("Request is a completion request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Chat completion request + if (rawRequest.TryGetProperty("messages", out _)) + { + Logger.LogDebug("Request is a chat completion request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Embedding request + if (rawRequest.TryGetProperty("input", out _) && + rawRequest.TryGetProperty("model", out _) && + !rawRequest.TryGetProperty("voice", out _)) + { + Logger.LogDebug("Request is an embedding request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Image generation request + if (rawRequest.TryGetProperty("prompt", out _) && + (rawRequest.TryGetProperty("size", out _) || rawRequest.TryGetProperty("n", out _))) + { + Logger.LogDebug("Request is an image generation request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Audio transcription request + if (rawRequest.TryGetProperty("file", out _)) + { + Logger.LogDebug("Request is an audio transcription request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Audio speech synthesis request + if (rawRequest.TryGetProperty("input", out _) && rawRequest.TryGetProperty("voice", out _)) + { + Logger.LogDebug("Request is an audio speech synthesis request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Fine-tuning request + if (rawRequest.TryGetProperty("training_file", out _)) + { + Logger.LogDebug("Request is a fine-tuning request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + Logger.LogDebug("Request is not an OpenAI request."); + return false; + } + catch (JsonException ex) + { + Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); + return false; + } + } + + public void Dispose() + { + _tracerProvider?.Dispose(); + _meterProvider?.Dispose(); + } +} diff --git a/dev-proxy-plugins/dev-proxy-plugins.csproj b/dev-proxy-plugins/dev-proxy-plugins.csproj index 718fbe6a..2e40b340 100644 --- a/dev-proxy-plugins/dev-proxy-plugins.csproj +++ b/dev-proxy-plugins/dev-proxy-plugins.csproj @@ -38,6 +38,14 @@ false runtime + + false + runtime + + + false + runtime + false runtime diff --git a/dev-proxy-plugins/packages.lock.json b/dev-proxy-plugins/packages.lock.json index a26068cc..8ff75f81 100644 --- a/dev-proxy-plugins/packages.lock.json +++ b/dev-proxy-plugins/packages.lock.json @@ -63,6 +63,26 @@ "resolved": "13.0.3", "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" }, + "OpenTelemetry": { + "type": "Direct", + "requested": "[1.12.0, )", + "resolved": "1.12.0", + "contentHash": "aIEu2O3xFOdwIVH0AJsIHPIMH1YuX18nzu7BHyaDNQ6NWSk4Zyrs9Pp6y8SATuSbvdtmvue4mj/QZ3838srbwA==", + "dependencies": { + "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging.Configuration": "9.0.0", + "OpenTelemetry.Api.ProviderBuilderExtensions": "1.12.0" + } + }, + "OpenTelemetry.Exporter.OpenTelemetryProtocol": { + "type": "Direct", + "requested": "[1.12.0, )", + "resolved": "1.12.0", + "contentHash": "7LzQSPhz5pNaL4xZgT3wkZODA1NLrEq3bet8KDHgtaJ9q+VNP7wmiZky8gQfMkB4FXuI/pevT8ZurL4p5997WA==", + "dependencies": { + "OpenTelemetry": "1.12.0" + } + }, "System.CommandLine": { "type": "Direct", "requested": "[2.0.0-beta4.22272.1, )", @@ -247,6 +267,15 @@ "resolved": "9.0.4", "contentHash": "ACtnvl3H3M/f8Z42980JxsNu7V9PPbzys4vBs83ZewnsgKd7JeYK18OMPo0g+MxAHrpgMrjmlinXDiaSRPcVnA==" }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "1K8P7XzuzX8W8pmXcZjcrqS6x5eSSdvhQohmcpgiQNY/HlDAlnrhR9dvlURfFz428A+RTCJpUyB+aKTA6AgVcQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0" + } + }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", "resolved": "9.0.4", @@ -288,6 +317,21 @@ "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.4" } }, + "Microsoft.Extensions.Logging.Configuration": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "H05HiqaNmg6GjH34ocYE9Wm1twm3Oz2aXZko8GTwGBzM7op2brpAA8pJ5yyD1OpS1mXUtModBYOlcZ/wXeWsSg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "9.0.0", + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", + "Microsoft.Extensions.Configuration.Binder": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging": "9.0.0", + "Microsoft.Extensions.Logging.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "9.0.0" + } + }, "Microsoft.Extensions.Options": { "type": "Transitive", "resolved": "9.0.4", @@ -297,6 +341,18 @@ "Microsoft.Extensions.Primitives": "9.0.4" } }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "Ob3FXsXkcSMQmGZi7qP07EQ39kZpSBlTcAZLbJLdI4FIf0Jug8biv2HTavWmnTirchctPlq9bl/26CXtQRguzA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", + "Microsoft.Extensions.Configuration.Binder": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Primitives": "9.0.0" + } + }, "Microsoft.Extensions.Primitives": { "type": "Transitive", "resolved": "9.0.4", @@ -375,6 +431,23 @@ "Newtonsoft.Json": "13.0.3" } }, + "OpenTelemetry.Api": { + "type": "Transitive", + "resolved": "1.12.0", + "contentHash": "Xt0qldi+iE2szGrM3jAqzEMEJd48YBtqI6mge0+ArXTZg3aTpRmyhL6CKKl3bLioaFSSVbBpEbPin8u6Z46Yrw==", + "dependencies": { + "System.Diagnostics.DiagnosticSource": "9.0.0" + } + }, + "OpenTelemetry.Api.ProviderBuilderExtensions": { + "type": "Transitive", + "resolved": "1.12.0", + "contentHash": "t6Vk1143BfiisCWYbRcyzkAuN6Aq5RkYtfOSMoqCIRMvtN9p1e1xzc0nWQ+fccNGOVgHn3aMK5xFn2+iWMcr8A==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "OpenTelemetry.Api": "1.12.0" + } + }, "SharpYaml": { "type": "Transitive", "resolved": "2.1.1", @@ -421,11 +494,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "6.0.1", - "contentHash": "KiLYDu2k2J82Q9BJpWiuQqCkFjRBWVq4jDzKKWawVi9KWzyD0XG3cmfX0vqTQlL14Wi9EufJrbL0+KCLTbqWiQ==", - "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } + "resolved": "9.0.0", + "contentHash": "ddppcFpnbohLWdYKr/ZeLZHmmI+DXFgZ3Snq+/E7SwcdW4UnvxmaugkwGywvGVWkHPGCSZjCP+MLzu23AL5SDw==" }, "System.Memory": { "type": "Transitive", diff --git a/dev-proxy/dev-proxy.csproj b/dev-proxy/dev-proxy.csproj index 29d9c7c7..ff9f0727 100644 --- a/dev-proxy/dev-proxy.csproj +++ b/dev-proxy/dev-proxy.csproj @@ -40,6 +40,8 @@ + + diff --git a/dev-proxy/packages.lock.json b/dev-proxy/packages.lock.json index 36c38e8b..67739f2f 100644 --- a/dev-proxy/packages.lock.json +++ b/dev-proxy/packages.lock.json @@ -112,6 +112,26 @@ "resolved": "13.0.3", "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" }, + "OpenTelemetry": { + "type": "Direct", + "requested": "[1.12.0, )", + "resolved": "1.12.0", + "contentHash": "aIEu2O3xFOdwIVH0AJsIHPIMH1YuX18nzu7BHyaDNQ6NWSk4Zyrs9Pp6y8SATuSbvdtmvue4mj/QZ3838srbwA==", + "dependencies": { + "Microsoft.Extensions.Diagnostics.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging.Configuration": "9.0.0", + "OpenTelemetry.Api.ProviderBuilderExtensions": "1.12.0" + } + }, + "OpenTelemetry.Exporter.OpenTelemetryProtocol": { + "type": "Direct", + "requested": "[1.12.0, )", + "resolved": "1.12.0", + "contentHash": "7LzQSPhz5pNaL4xZgT3wkZODA1NLrEq3bet8KDHgtaJ9q+VNP7wmiZky8gQfMkB4FXuI/pevT8ZurL4p5997WA==", + "dependencies": { + "OpenTelemetry": "1.12.0" + } + }, "Swashbuckle.AspNetCore": { "type": "Direct", "requested": "[8.1.1, )", @@ -294,6 +314,15 @@ "resolved": "9.0.4", "contentHash": "ACtnvl3H3M/f8Z42980JxsNu7V9PPbzys4vBs83ZewnsgKd7JeYK18OMPo0g+MxAHrpgMrjmlinXDiaSRPcVnA==" }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "1K8P7XzuzX8W8pmXcZjcrqS6x5eSSdvhQohmcpgiQNY/HlDAlnrhR9dvlURfFz428A+RTCJpUyB+aKTA6AgVcQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0" + } + }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", "resolved": "9.0.4", @@ -463,6 +492,23 @@ "Newtonsoft.Json": "13.0.3" } }, + "OpenTelemetry.Api": { + "type": "Transitive", + "resolved": "1.12.0", + "contentHash": "Xt0qldi+iE2szGrM3jAqzEMEJd48YBtqI6mge0+ArXTZg3aTpRmyhL6CKKl3bLioaFSSVbBpEbPin8u6Z46Yrw==", + "dependencies": { + "System.Diagnostics.DiagnosticSource": "9.0.0" + } + }, + "OpenTelemetry.Api.ProviderBuilderExtensions": { + "type": "Transitive", + "resolved": "1.12.0", + "contentHash": "t6Vk1143BfiisCWYbRcyzkAuN6Aq5RkYtfOSMoqCIRMvtN9p1e1xzc0nWQ+fccNGOVgHn3aMK5xFn2+iWMcr8A==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "OpenTelemetry.Api": "1.12.0" + } + }, "SharpYaml": { "type": "Transitive", "resolved": "2.1.1", @@ -530,11 +576,8 @@ }, "System.Diagnostics.DiagnosticSource": { "type": "Transitive", - "resolved": "6.0.1", - "contentHash": "KiLYDu2k2J82Q9BJpWiuQqCkFjRBWVq4jDzKKWawVi9KWzyD0XG3cmfX0vqTQlL14Wi9EufJrbL0+KCLTbqWiQ==", - "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - } + "resolved": "9.0.0", + "contentHash": "ddppcFpnbohLWdYKr/ZeLZHmmI+DXFgZ3Snq+/E7SwcdW4UnvxmaugkwGywvGVWkHPGCSZjCP+MLzu23AL5SDw==" }, "System.Memory": { "type": "Transitive", diff --git a/schemas/v0.28.0/openaitelemetryplugin.pricesfile.schema.json b/schemas/v0.28.0/openaitelemetryplugin.pricesfile.schema.json new file mode 100644 index 00000000..c83a3362 --- /dev/null +++ b/schemas/v0.28.0/openaitelemetryplugin.pricesfile.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenAI Telemetry Plugin language model prices file schema", + "description": "Schema for the language model prices file used by the OpenAI Telemetry plugin.", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "The JSON schema reference for validation." + }, + "prices": { + "type": "object", + "description": "Map of model names to their pricing information.", + "additionalProperties": { + "type": "object", + "properties": { + "input": { + "type": "number", + "description": "The price per million tokens for input/prompt tokens." + }, + "output": { + "type": "number", + "description": "The price per million tokens for output/completion tokens." + } + }, + "required": [ + "input", + "output" + ] + } + } + }, + "required": [ + "prices" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/schemas/v0.28.0/openaitelemetryplugin.schema.json b/schemas/v0.28.0/openaitelemetryplugin.schema.json new file mode 100644 index 00000000..e805c7cf --- /dev/null +++ b/schemas/v0.28.0/openaitelemetryplugin.schema.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "OpenAI Telemetry Plugin", + "description": "Settings for the OpenAI Telemetry plugin which captures OpenAI API calls and emits OpenTelemetry information.", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "The JSON schema reference for validation." + }, + "application": { + "type": "string", + "description": "The name of the application using the OpenTelemetry plugin.", + "default": "default" + }, + "currency": { + "type": "string", + "description": "The currency used for cost calculations.", + "default": "USD" + }, + "environment": { + "type": "string", + "description": "The environment in which the application is running (e.g., production, staging, development).", + "default": "development" + }, + "exporterEndpoint": { + "type": "string", + "description": "The endpoint of the OpenTelemetry collector to send information to.", + "default": "http://localhost:4318" + }, + "includeCompletion": { + "type": "boolean", + "description": "Whether to include the completion in the OpenTelemetry span. Disable for privacy or security concerns.", + "default": true + }, + "includeCosts": { + "type": "boolean", + "description": "Whether to calculate and include cost information in the spans. Requires prices data.", + "default": true + }, + "includePrompt": { + "type": "boolean", + "description": "Whether to include the prompt in the OpenTelemetry span. Disable for privacy or security concerns.", + "default": true + }, + "pricesFile": { + "type": "string", + "description": "Path to the JSON file containing prices data for language models." + } + }, + "additionalProperties": false +}